diff options
201 files changed, 16623 insertions, 2042 deletions
diff --git a/.gitignore b/.gitignore index 319c5bfdd..004f7d759 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,13 @@ *.lo *.o *.so +*.lo +*.o +*.la *.a *.swp +*.save +*.cmd *~ *.gch dist* @@ -21,6 +26,9 @@ mert/kbmira misc/processLexicalTable misc/processPhraseTable misc/queryLexicalTable +mira/mira +mira/Makefile +mira/Makefile.in misc/queryPhraseTable moses-chart-cmd/src/moses_chart moses-cmd/src/checkplf @@ -50,11 +50,11 @@ # # debug-symbols=on|off include (default) or exclude debugging # information also known as -g -# # --notrace compiles without TRACE macros # # --enable-boost-pool uses Boost pools for the memory SCFG table # +# --enable-mpi switch on mpi # --without-libsegfault does not link with libSegFault # # --max-kenlm-order maximum ngram order that kenlm can process (default 6) @@ -78,6 +78,17 @@ if [ option.get "with-tcmalloc" : : "yes" ] { requirements += <library>tcmalloc ; } +if [ option.get "enable-mpi" : : "yes" ] { + import mpi ; + using mpi ; + external-lib boost_mpi ; + external-lib boost_serialization ; + requirements += <define>MPI_ENABLE ; + requirements += <library>mpi ; + requirements += <library>boost_mpi ; + requirements += <library>boost_serialization ; +} + requirements += [ option.get "notrace" : <define>TRACE_ENABLE=1 ] ; requirements += [ option.get "enable-boost-pool" : : <define>USE_BOOST_POOL ] ; @@ -101,9 +112,9 @@ project : requirements ; #Add directories here if you want their incidental targets too (i.e. tests). -build-projects util lm mert moses-cmd/src moses-chart-cmd/src scripts regression-testing contrib/relent-filter/src ; +build-projects util lm mert moses-cmd/src moses-chart-cmd/src mira scripts regression-testing ; -alias programs : lm//query lm//build_binary lm//kenlm_max_order moses-chart-cmd/src//moses_chart moses-cmd/src//programs OnDiskPt//CreateOnDiskPt OnDiskPt//queryOnDiskPt mert//programs contrib/server//mosesserver misc//programs symal phrase-extract phrase-extract//lexical-reordering phrase-extract//extract-ghkm phrase-extract//pcfg-extract phrase-extract//pcfg-score biconcor contrib/relent-filter/src//calcDivergence ; +alias programs : lm//query lm//build_binary lm//kenlm_max_order moses-chart-cmd/src//moses_chart moses-cmd/src//programs OnDiskPt//CreateOnDiskPt OnDiskPt//queryOnDiskPt mert//programs contrib/server//mosesserver misc//programs mira//programs symal phrase-extract phrase-extract//lexical-reordering phrase-extract//extract-ghkm phrase-extract//pcfg-extract phrase-extract//pcfg-score biconcor ; install-bin-libs programs ; install-headers headers-base : [ path.glob-tree biconcor contrib lm mert misc moses-chart-cmd moses-cmd OnDiskPt phrase-extract symal util : *.hh *.h ] : . ; diff --git a/OnDiskPt/Main.cpp b/OnDiskPt/Main.cpp index e9e51c58d..acff2d405 100644 --- a/OnDiskPt/Main.cpp +++ b/OnDiskPt/Main.cpp @@ -56,7 +56,6 @@ int main (int argc, char * const argv[]) const string filePath = argv[6] ,destPath = argv[7]; - Moses::InputFileStream inStream(filePath); OnDiskWrapper onDiskWrapper; @@ -78,10 +77,10 @@ int main (int argc, char * const argv[]) std::vector<float> misc(1); SourcePhrase sourcePhrase; TargetPhrase *targetPhrase = new TargetPhrase(numScores); - Tokenize(sourcePhrase, *targetPhrase, line, onDiskWrapper, numScores, misc); + OnDiskPt::PhrasePtr spShort = Tokenize(sourcePhrase, *targetPhrase, line, onDiskWrapper, numScores, misc); assert(misc.size() == onDiskWrapper.GetNumCounts()); - rootNode.AddTargetPhrase(sourcePhrase, targetPhrase, onDiskWrapper, tableLimit, misc); + rootNode.AddTargetPhrase(sourcePhrase, targetPhrase, onDiskWrapper, tableLimit, misc, spShort); } rootNode.Save(onDiskWrapper, 0, tableLimit); @@ -106,7 +105,7 @@ bool Flush(const OnDiskPt::SourcePhrase *prevSourcePhrase, const OnDiskPt::Sourc return ret; } -void Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhrase, char *line, OnDiskWrapper &onDiskWrapper, int numScores, vector<float> &misc) +OnDiskPt::PhrasePtr Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhrase, char *line, OnDiskWrapper &onDiskWrapper, int numScores, vector<float> &misc) { size_t scoreInd = 0; @@ -119,13 +118,17 @@ void Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhrase, char *line 4 = count */ char *tok = strtok (line," "); + OnDiskPt::PhrasePtr out(new Phrase()); while (tok != NULL) { if (0 == strcmp(tok, "|||")) { ++stage; } else { switch (stage) { case 0: { - Tokenize(sourcePhrase, tok, true, true, onDiskWrapper); + WordPtr w = Tokenize(sourcePhrase, tok, true, true, onDiskWrapper); + if (w != NULL) + out->AddWord(w); + break; } case 1: { @@ -139,19 +142,34 @@ void Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhrase, char *line break; } case 3: { - targetPhrase.Create1AlignFromString(tok); + //targetPhrase.Create1AlignFromString(tok); + targetPhrase.CreateAlignFromString(tok); break; } case 4: ++stage; break; - case 5: { + /* case 5: { // count info. Only store the 2nd one float val = Moses::Scan<float>(tok); misc[0] = val; ++stage; break; + }*/ + case 5: { + // count info. Only store the 2nd one + //float val = Moses::Scan<float>(tok); + //misc[0] = val; + ++stage; + break; } + case 6: { + // store only the 3rd one (rule count) + float val = Moses::Scan<float>(tok); + misc[0] = val; + ++stage; + break; + } default: assert(false); break; @@ -163,10 +181,10 @@ void Tokenize(SourcePhrase &sourcePhrase, TargetPhrase &targetPhrase, char *line assert(scoreInd == numScores); targetPhrase.SortAlign(); - + return out; } // Tokenize() -void Tokenize(OnDiskPt::Phrase &phrase +OnDiskPt::WordPtr Tokenize(OnDiskPt::Phrase &phrase , const std::string &token, bool addSourceNonTerm, bool addTargetNonTerm , OnDiskPt::OnDiskWrapper &onDiskWrapper) { @@ -180,6 +198,7 @@ void Tokenize(OnDiskPt::Phrase &phrase nonTerm = comStr == 0; } + OnDiskPt::WordPtr out; if (nonTerm) { // non-term size_t splitPos = token.find_first_of("[", 2); @@ -187,31 +206,35 @@ void Tokenize(OnDiskPt::Phrase &phrase if (splitPos == string::npos) { // lhs - only 1 word - Word *word = new Word(); + WordPtr word(new Word()); word->CreateFromString(wordStr, onDiskWrapper.GetVocab()); phrase.AddWord(word); } else { // source & target non-terms if (addSourceNonTerm) { - Word *word = new Word(); + WordPtr word(new Word()); word->CreateFromString(wordStr, onDiskWrapper.GetVocab()); - phrase.AddWord(word); + phrase.AddWord(word); } wordStr = token.substr(splitPos, tokSize - splitPos); if (addTargetNonTerm) { - Word *word = new Word(); + WordPtr word(new Word()); word->CreateFromString(wordStr, onDiskWrapper.GetVocab()); phrase.AddWord(word); + out = word; } } } else { // term - Word *word = new Word(); + WordPtr word(new Word()); word->CreateFromString(token, onDiskWrapper.GetVocab()); phrase.AddWord(word); + out = word; } + + return out; } void InsertTargetNonTerminals(std::vector<std::string> &sourceToks, const std::vector<std::string> &targetToks, const ::AlignType &alignments) diff --git a/OnDiskPt/Main.h b/OnDiskPt/Main.h index 41a24a239..c8cfcb1dd 100644 --- a/OnDiskPt/Main.h +++ b/OnDiskPt/Main.h @@ -25,10 +25,10 @@ typedef std::pair<size_t, size_t> AlignPair; typedef std::vector<AlignPair> AlignType; -void Tokenize(OnDiskPt::Phrase &phrase +OnDiskPt::WordPtr Tokenize(OnDiskPt::Phrase &phrase , const std::string &token, bool addSourceNonTerm, bool addTargetNonTerm , OnDiskPt::OnDiskWrapper &onDiskWrapper); -void Tokenize(OnDiskPt::SourcePhrase &sourcePhrase, OnDiskPt::TargetPhrase &targetPhrase +OnDiskPt::PhrasePtr Tokenize(OnDiskPt::SourcePhrase &sourcePhrase, OnDiskPt::TargetPhrase &targetPhrase , char *line, OnDiskPt::OnDiskWrapper &onDiskWrapper , int numScores , std::vector<float> &misc); diff --git a/OnDiskPt/Phrase.cpp b/OnDiskPt/Phrase.cpp index dc289a81a..73d2dbac9 100644 --- a/OnDiskPt/Phrase.cpp +++ b/OnDiskPt/Phrase.cpp @@ -27,27 +27,13 @@ using namespace std; namespace OnDiskPt { -Phrase::Phrase(const Phrase ©) - :m_words(copy.GetSize()) -{ - for (size_t pos = 0; pos < copy.GetSize(); ++pos) { - const Word &oldWord = copy.GetWord(pos); - Word *newWord = new Word(oldWord); - m_words[pos] = newWord; - } -} - -Phrase::~Phrase() -{ - Moses::RemoveAllInColl(m_words); -} -void Phrase::AddWord(Word *word) +void Phrase::AddWord(WordPtr word) { m_words.push_back(word); } -void Phrase::AddWord(Word *word, size_t pos) +void Phrase::AddWord(WordPtr word, size_t pos) { CHECK(pos < m_words.size()); m_words.insert(m_words.begin() + pos + 1, word); diff --git a/OnDiskPt/Phrase.h b/OnDiskPt/Phrase.h index 5ffa56d6b..e785be823 100644 --- a/OnDiskPt/Phrase.h +++ b/OnDiskPt/Phrase.h @@ -20,12 +20,14 @@ ***********************************************************************/ #include <vector> #include <iostream> +#include <boost/shared_ptr.hpp> #include "Word.h" namespace OnDiskPt { class Vocab; + /** A contiguous phrase. SourcePhrase & TargetPhrase inherit from this and add the on-disk functionality */ class Phrase @@ -33,16 +35,14 @@ class Phrase friend std::ostream& operator<<(std::ostream&, const Phrase&); protected: - std::vector<Word*> m_words; + std::vector<WordPtr> m_words; public: Phrase() {} - Phrase(const Phrase ©); - virtual ~Phrase(); - void AddWord(Word *word); - void AddWord(Word *word, size_t pos); + void AddWord(WordPtr word); + void AddWord(WordPtr word, size_t pos); const Word &GetWord(size_t pos) const { return *m_words[pos]; @@ -59,4 +59,6 @@ public: bool operator==(const Phrase &compare) const; }; +typedef boost::shared_ptr<Phrase> PhrasePtr; + } diff --git a/OnDiskPt/PhraseNode.cpp b/OnDiskPt/PhraseNode.cpp index 9b2c0bbc4..5ced697b4 100644 --- a/OnDiskPt/PhraseNode.cpp +++ b/OnDiskPt/PhraseNode.cpp @@ -58,7 +58,7 @@ PhraseNode::PhraseNode(UINT64 filePos, OnDiskWrapper &onDiskWrapper) CHECK(filePos == (UINT64)file.tellg()); file.read((char*) &m_numChildrenLoad, sizeof(UINT64)); - + size_t memAlloc = GetNodeSize(m_numChildrenLoad, onDiskWrapper.GetSourceWordSize(), countSize); m_memLoad = (char*) malloc(memAlloc); @@ -160,15 +160,15 @@ void PhraseNode::Save(OnDiskWrapper &onDiskWrapper, size_t pos, size_t tableLimi void PhraseNode::AddTargetPhrase(const SourcePhrase &sourcePhrase, TargetPhrase *targetPhrase , OnDiskWrapper &onDiskWrapper, size_t tableLimit - , const std::vector<float> &counts) + , const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort) { - AddTargetPhrase(0, sourcePhrase, targetPhrase, onDiskWrapper, tableLimit, counts); + AddTargetPhrase(0, sourcePhrase, targetPhrase, onDiskWrapper, tableLimit, counts, spShort); } void PhraseNode::AddTargetPhrase(size_t pos, const SourcePhrase &sourcePhrase , TargetPhrase *targetPhrase, OnDiskWrapper &onDiskWrapper - , size_t tableLimit, const std::vector<float> &counts) -{ + , size_t tableLimit, const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort) +{ size_t phraseSize = sourcePhrase.GetSize(); if (pos < phraseSize) { const Word &word = sourcePhrase.GetWord(pos); @@ -185,10 +185,12 @@ void PhraseNode::AddTargetPhrase(size_t pos, const SourcePhrase &sourcePhrase m_currChild = &node; } - node.AddTargetPhrase(pos + 1, sourcePhrase, targetPhrase, onDiskWrapper, tableLimit, counts); + // keep searching for target phrase node.. + node.AddTargetPhrase(pos + 1, sourcePhrase, targetPhrase, onDiskWrapper, tableLimit, counts, spShort); } else { // drilled down to the right node m_counts = counts; + targetPhrase->SetSourcePhrase(spShort); m_targetPhraseColl.AddTargetPhrase(targetPhrase); } } diff --git a/OnDiskPt/PhraseNode.h b/OnDiskPt/PhraseNode.h index c749692e0..fbd20ce36 100644 --- a/OnDiskPt/PhraseNode.h +++ b/OnDiskPt/PhraseNode.h @@ -23,6 +23,7 @@ #include <map> #include "Word.h" #include "TargetPhraseCollection.h" +#include "Phrase.h" namespace OnDiskPt { @@ -51,8 +52,8 @@ protected: void AddTargetPhrase(size_t pos, const SourcePhrase &sourcePhrase , TargetPhrase *targetPhrase, OnDiskWrapper &onDiskWrapper - , size_t tableLimit, const std::vector<float> &counts); - size_t ReadChild(Word &wordFound, UINT64 &childFilePos, const char *mem) const; + , size_t tableLimit, const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort); + size_t ReadChild(Word &wordFound, UINT64 &childFilePos, const char *mem) const; void GetChild(Word &wordFound, UINT64 &childFilePos, size_t ind, OnDiskWrapper &onDiskWrapper) const; public: @@ -67,7 +68,7 @@ public: void AddTargetPhrase(const SourcePhrase &sourcePhrase, TargetPhrase *targetPhrase , OnDiskWrapper &onDiskWrapper, size_t tableLimit - , const std::vector<float> &counts); + , const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort); UINT64 GetFilePos() const { return m_filePos; diff --git a/OnDiskPt/SourcePhrase.h b/OnDiskPt/SourcePhrase.h index 23563b0bb..e0c510e42 100644 --- a/OnDiskPt/SourcePhrase.h +++ b/OnDiskPt/SourcePhrase.h @@ -34,4 +34,5 @@ protected: public: }; + } diff --git a/OnDiskPt/TargetPhrase.cpp b/OnDiskPt/TargetPhrase.cpp index d923df8fd..e7123af3b 100644 --- a/OnDiskPt/TargetPhrase.cpp +++ b/OnDiskPt/TargetPhrase.cpp @@ -27,6 +27,8 @@ #include "TargetPhrase.h" #include "OnDiskWrapper.h" +#include <boost/algorithm/string.hpp> + using namespace std; namespace OnDiskPt @@ -48,7 +50,7 @@ TargetPhrase::~TargetPhrase() { } -void TargetPhrase::SetLHS(Word *lhs) +void TargetPhrase::SetLHS(WordPtr lhs) { AddWord(lhs); } @@ -61,6 +63,18 @@ void TargetPhrase::Create1AlignFromString(const std::string &align1Str) m_align.push_back(pair<size_t, size_t>(alignPoints[0], alignPoints[1]) ); } +void TargetPhrase::CreateAlignFromString(const std::string &alignStr) +{ + vector<std::string> alignPairs; + boost::split(alignPairs, alignStr, boost::is_any_of("\t ")); + for (size_t i = 0; i < alignPairs.size(); ++i) { + vector<size_t> alignPoints; + Moses::Tokenize<size_t>(alignPoints, alignPairs[i], "-"); + m_align.push_back(pair<size_t, size_t>(alignPoints[0], alignPoints[1]) ); + } +} + + void TargetPhrase::SetScore(float score, size_t ind) { CHECK(ind < m_scores.size()); @@ -84,9 +98,16 @@ char *TargetPhrase::WriteToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed) { size_t phraseSize = GetSize(); size_t targetWordSize = onDiskWrapper.GetTargetWordSize(); - + + const PhrasePtr sp = GetSourcePhrase(); + size_t spSize = sp->GetSize(); + size_t sourceWordSize = onDiskWrapper.GetSourceWordSize(); + size_t memNeeded = sizeof(UINT64) // num of words - + targetWordSize * phraseSize; // actual words. lhs as last words + + targetWordSize * phraseSize // actual words. lhs as last words + + sizeof(UINT64) // num source words + + sourceWordSize * spSize; // actual source words + memUsed = 0; UINT64 *mem = (UINT64*) malloc(memNeeded); @@ -101,6 +122,17 @@ char *TargetPhrase::WriteToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed) memUsed += word.WriteToMemory((char*) currPtr); } + // write size of source phrase and all source words + char *currPtr = (char*)mem + memUsed; + UINT64 *memTmp = (UINT64*) currPtr; + memTmp[0] = spSize; + memUsed += sizeof(UINT64); + for (size_t pos = 0; pos < spSize; ++pos) { + const Word &word = sp->GetWord(pos); + char *currPtr = (char*)mem + memUsed; + memUsed += word.WriteToMemory((char*) currPtr); + } + CHECK(memUsed == memNeeded); return (char *) mem; } @@ -143,9 +175,10 @@ char *TargetPhrase::WriteOtherInfoToMemory(OnDiskWrapper &onDiskWrapper, size_t // phrase id memcpy(mem, &m_filePos, sizeof(UINT64)); memUsed += sizeof(UINT64); - + // align - memUsed += WriteAlignToMemory(mem + memUsed); + size_t tmp = WriteAlignToMemory(mem + memUsed); + memUsed += tmp; // scores memUsed += WriteScoresToMemory(mem + memUsed); @@ -191,7 +224,7 @@ size_t TargetPhrase::WriteScoresToMemory(char *mem) const } -Moses::TargetPhrase *TargetPhrase::ConvertToMoses(const std::vector<Moses::FactorType> & /*inputFactors */ +Moses::TargetPhrase *TargetPhrase::ConvertToMoses(const std::vector<Moses::FactorType> & inputFactors , const std::vector<Moses::FactorType> &outputFactors , const Vocab &vocab , const Moses::PhraseDictionary &phraseDict @@ -214,15 +247,27 @@ Moses::TargetPhrase *TargetPhrase::ConvertToMoses(const std::vector<Moses::Facto ret->SetScoreChart(phraseDict.GetFeature(), m_scores, weightT, lmList, wpProducer); // alignments + int indicator[m_align.size()]; + int index = 0; std::set<std::pair<size_t, size_t> > alignmentInfo; + const PhrasePtr sp = GetSourcePhrase(); for (size_t ind = 0; ind < m_align.size(); ++ind) { const std::pair<size_t, size_t> &entry = m_align[ind]; alignmentInfo.insert(entry); + size_t sourcePos = entry.first; + indicator[index++] = sp->GetWord(sourcePos).IsNonTerminal() ? 1: 0; } - ret->SetAlignmentInfo(alignmentInfo); + ret->SetAlignmentInfo(alignmentInfo, indicator); GetWord(GetSize() - 1).ConvertToMoses(outputFactors, vocab, ret->MutableTargetLHS()); - + + // set source phrase + Moses::Phrase mosesSP(Moses::Input); + for (size_t pos = 0; pos < sp->GetSize(); ++pos) { + sp->GetWord(pos).ConvertToMoses(inputFactors, vocab, mosesSP.AddWord()); + } + ret->SetSourcePhrase(mosesSP); + return ret; } @@ -255,10 +300,23 @@ UINT64 TargetPhrase::ReadFromFile(std::fstream &fileTP) bytesRead += sizeof(UINT64); for (size_t ind = 0; ind < numWords; ++ind) { - Word *word = new Word(); + WordPtr word(new Word()); bytesRead += word->ReadFromFile(fileTP); AddWord(word); } + + // read source words + UINT64 numSourceWords; + fileTP.read((char*) &numSourceWords, sizeof(UINT64)); + bytesRead += sizeof(UINT64); + + PhrasePtr sp(new SourcePhrase()); + for (size_t ind = 0; ind < numSourceWords; ++ind) { + WordPtr word( new Word()); + bytesRead += word->ReadFromFile(fileTP); + sp->AddWord(word); + } + SetSourcePhrase(sp); return bytesRead; } diff --git a/OnDiskPt/TargetPhrase.h b/OnDiskPt/TargetPhrase.h index 4f917095c..37fd0f526 100644 --- a/OnDiskPt/TargetPhrase.h +++ b/OnDiskPt/TargetPhrase.h @@ -24,6 +24,7 @@ #include <vector> #include "Word.h" #include "Phrase.h" +#include "SourcePhrase.h" namespace Moses { @@ -50,6 +51,7 @@ class TargetPhrase: public Phrase friend std::ostream& operator<<(std::ostream&, const TargetPhrase&); protected: AlignType m_align; + PhrasePtr m_sourcePhrase; std::vector<float> m_scores; UINT64 m_filePos; @@ -65,9 +67,17 @@ public: TargetPhrase(const TargetPhrase ©); virtual ~TargetPhrase(); - void SetLHS(Word *lhs); + void SetSourcePhrase(PhrasePtr p) { + m_sourcePhrase = p; + } + const PhrasePtr GetSourcePhrase() const { + return m_sourcePhrase; + } + + void SetLHS(WordPtr lhs); void Create1AlignFromString(const std::string &align1Str); + void CreateAlignFromString(const std::string &align1Str); void SetScore(float score, size_t ind); const AlignType &GetAlign() const { diff --git a/OnDiskPt/TargetPhraseCollection.cpp b/OnDiskPt/TargetPhraseCollection.cpp index 20e5bbb48..11ff7cea6 100644 --- a/OnDiskPt/TargetPhraseCollection.cpp +++ b/OnDiskPt/TargetPhraseCollection.cpp @@ -82,7 +82,7 @@ void TargetPhraseCollection::Save(OnDiskWrapper &onDiskWrapper) CollType::iterator iter; for (iter = m_coll.begin(); iter != m_coll.end(); ++iter) { // save phrase - TargetPhrase &targetPhrase = **iter; + TargetPhrase &targetPhrase = **iter; targetPhrase.Save(onDiskWrapper); // save coll @@ -154,8 +154,9 @@ void TargetPhraseCollection::ReadFromFile(size_t tableLimit, UINT64 filePos, OnD { fstream &fileTPColl = onDiskWrapper.GetFileTargetColl(); fstream &fileTP = onDiskWrapper.GetFileTargetInd(); - + size_t numScores = onDiskWrapper.GetNumScores(); + UINT64 numPhrases; @@ -167,9 +168,9 @@ void TargetPhraseCollection::ReadFromFile(size_t tableLimit, UINT64 filePos, OnD numPhrases = std::min(numPhrases, (UINT64) tableLimit); currFilePos += sizeof(UINT64); - + for (size_t ind = 0; ind < numPhrases; ++ind) { - TargetPhrase *tp = new TargetPhrase(numScores); + TargetPhrase *tp = new TargetPhrase(numScores); UINT64 sizeOtherInfo = tp->ReadOtherInfoFromFile(currFilePos, fileTPColl); tp->ReadFromFile(fileTP); diff --git a/OnDiskPt/Vocab.cpp b/OnDiskPt/Vocab.cpp index a6ab68cdd..5de620b75 100644 --- a/OnDiskPt/Vocab.cpp +++ b/OnDiskPt/Vocab.cpp @@ -43,7 +43,7 @@ bool Vocab::Load(OnDiskWrapper &onDiskWrapper) // create lookup // assume contiguous vocab id m_lookup.resize(m_vocabColl.size() + 1); - m_nextId = m_lookup.size(); + m_nextId = m_lookup.size(); CollType::const_iterator iter; for (iter = m_vocabColl.begin(); iter != m_vocabColl.end(); ++iter) { diff --git a/OnDiskPt/Word.h b/OnDiskPt/Word.h index 497684ecf..8c65cf7e5 100644 --- a/OnDiskPt/Word.h +++ b/OnDiskPt/Word.h @@ -22,6 +22,7 @@ #include <vector> #include <iostream> #include <fstream> +#include <boost/shared_ptr.hpp> #include "Vocab.h" namespace Moses @@ -82,5 +83,7 @@ public: bool operator==(const Word &compare) const; }; + +typedef boost::shared_ptr<Word> WordPtr; } diff --git a/OnDiskPt/queryOnDiskPt.cpp b/OnDiskPt/queryOnDiskPt.cpp index 9a2d97680..97c100a29 100644 --- a/OnDiskPt/queryOnDiskPt.cpp +++ b/OnDiskPt/queryOnDiskPt.cpp @@ -38,20 +38,20 @@ void Tokenize(OnDiskPt::Phrase &phrase if (splitPos == string::npos) { // lhs - only 1 word - Word *word = new Word(); + WordPtr word (new Word()); word->CreateFromString(wordStr, onDiskWrapper.GetVocab()); phrase.AddWord(word); } else { // source & target non-terms if (addSourceNonTerm) { - Word *word = new Word(); + WordPtr word( new Word()); word->CreateFromString(wordStr, onDiskWrapper.GetVocab()); phrase.AddWord(word); } wordStr = token.substr(splitPos, tokSize - splitPos); if (addTargetNonTerm) { - Word *word = new Word(); + WordPtr word(new Word()); word->CreateFromString(wordStr, onDiskWrapper.GetVocab()); phrase.AddWord(word); } @@ -59,7 +59,7 @@ void Tokenize(OnDiskPt::Phrase &phrase } } else { // term - Word *word = new Word(); + WordPtr word(new Word()); word->CreateFromString(token, onDiskWrapper.GetVocab()); phrase.AddWord(word); } diff --git a/contrib/other-builds/OnDiskPt.xcodeproj/project.pbxproj b/contrib/other-builds/OnDiskPt.xcodeproj/project.pbxproj index 138ef3359..80b2db5b6 100644 --- a/contrib/other-builds/OnDiskPt.xcodeproj/project.pbxproj +++ b/contrib/other-builds/OnDiskPt.xcodeproj/project.pbxproj @@ -226,6 +226,7 @@ GCC_OPTIMIZATION_LEVEL = 0; GCC_WARN_ABOUT_RETURN_TYPE = YES; GCC_WARN_UNUSED_VARIABLE = YES; + HEADER_SEARCH_PATHS = /usr/local/include; ONLY_ACTIVE_ARCH = YES; PREBINDING = NO; SDKROOT = macosx10.6; @@ -239,6 +240,7 @@ GCC_C_LANGUAGE_STANDARD = gnu99; GCC_WARN_ABOUT_RETURN_TYPE = YES; GCC_WARN_UNUSED_VARIABLE = YES; + HEADER_SEARCH_PATHS = /usr/local/include; ONLY_ACTIVE_ARCH = YES; PREBINDING = NO; SDKROOT = macosx10.6; diff --git a/contrib/server/sgclient_weightUpdate.perl b/contrib/server/sgclient_weightUpdate.perl new file mode 100755 index 000000000..736ab9b84 --- /dev/null +++ b/contrib/server/sgclient_weightUpdate.perl @@ -0,0 +1,74 @@ +#!/usr/bin/perl -w +use strict; +use Frontier::Client; + +my $output_suffix = $ARGV[0]; +$output_suffix = "" if (not $output_suffix); + +my $port = "50015"; +my $url = "http://localhost:".$port."/RPC2"; +my $server = Frontier::Client->new('url' => $url, 'encoding' => 'UTF-8'); +my $verbose=0; + +my $translations="translations$output_suffix.out"; +open TR, ">:utf8", $translations; +my $sg_out="searchGraph$output_suffix.out"; +open SG, ">:utf8", $sg_out; + +my $i=0; +while (my $text = <STDIN>) +{ + my $date = `date`; + chop($date); + print "[$date] sentence $i: translate\n" if $verbose; + + # update weights + my $core_weights = "0.0314787,-0.138354,1,0.0867223,0.0349965,0.104774,0.0607203,0.0516889,0.113694,0.0947218,0.0642702,0.0385324,0.0560749,0.0434684,0.0805031"; + #my $core_weights = "0.0314787,-0.138354,1,0.0867223,0.0349965,0.104774,0.0607203,0.0516889,0.113694,0.0947218,0.0642702,0.0385324,0.0560749,0.0434684,0.0805031,0"; + #my $sparse_weights = "pp_dummy~dummy=0.001"; + my $sparse_weights = ""; + my %param = ("core-weights" => $core_weights, "sparse-weights" => $sparse_weights); + $server->call("setWeights",(\%param)); + + #my $core_weight_update = "0.1,0.1,0,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1"; + #my $sparse_weight_update = "pp_dummy~dummy=0.1"; + #my %param_update = ("core-weights" => $core_weight_update, "sparse-weights" => $sparse_weight_update); + #$server->call("addWeights",(\%param_update)); + + # translate + #my %param = ("text" => $server->string($SENTENCE[$i]) , "sg" => "true"); + %param = ("text" => $text, "id" => $i, "sg" => "true"); + my $result = $server->call("translate",(\%param)); + + $date = `date`; + chop($date); + print "[$date] sentence $i: process translation\n" if $verbose; + + # process translation + my $mt_output = $result->{'text'}; + $mt_output =~ s/\|\S+//g; # no multiple factors, only first + print "sentence $i >> $translations \n"; + print TR $mt_output."\n"; + + # print out search graph + print "sentence $i >> $sg_out \n"; + my $sg_ref = $result->{'sg'}; + foreach my $sgn (@$sg_ref) { + # print out in extended format + if ($sgn->{hyp} eq 0) { + print SG "$i hyp=$sgn->{'hyp'} stack=$sgn->{'stack'} forward=$sgn->{'forward'} fscore=$sgn->{'fscore'} \n"; + } + else { + print SG "$i hyp=$sgn->{'hyp'} stack=$sgn->{'stack'} back=$sgn->{'back'} score=$sgn->{'score'} transition=$sgn->{'transition'} "; + if ($sgn->{"recombined"}) { + print SG "recombined=$sgn->{'recombined'} "; + } + print SG "forward=$sgn->{'forward'} fscore=$sgn->{'fscore'} covered=$sgn->{'cover-start'}-$sgn->{'cover-end'} "; + print SG "scores=\"$sgn->{'scores'}\" src-phrase=\"$sgn->{'src-phrase'}\" tgt-phrase=\"$sgn->{'tgt-phrase'}\" \n"; + } + } + + ++$i; +} + +close(SG); diff --git a/contrib/web/bin/detokenizer.perl b/contrib/web/bin/detokenizer.perl index 4d53aeddd..4d53aeddd 100644..100755 --- a/contrib/web/bin/detokenizer.perl +++ b/contrib/web/bin/detokenizer.perl diff --git a/contrib/web/bin/start-daemon-cluster.pl b/contrib/web/bin/start-daemon-cluster.pl index 7bf87842c..7bf87842c 100644..100755 --- a/contrib/web/bin/start-daemon-cluster.pl +++ b/contrib/web/bin/start-daemon-cluster.pl diff --git a/contrib/web/bin/tokenizer.perl b/contrib/web/bin/tokenizer.perl index 4e1f5ac05..4e1f5ac05 100644..100755 --- a/contrib/web/bin/tokenizer.perl +++ b/contrib/web/bin/tokenizer.perl diff --git a/jam-files/engine/bump_version.py b/jam-files/engine/bump_version.py index 9423c4c77..9423c4c77 100644..100755 --- a/jam-files/engine/bump_version.py +++ b/jam-files/engine/bump_version.py diff --git a/mert/BleuScorer.cpp b/mert/BleuScorer.cpp index 50227352c..b9cae5a15 100644 --- a/mert/BleuScorer.cpp +++ b/mert/BleuScorer.cpp @@ -12,6 +12,8 @@ #include "Ngram.h" #include "Reference.h" #include "Util.h" +#include "ScoreDataIterator.h" +#include "FeatureDataIterator.h" #include "Vocabulary.h" using namespace std; @@ -278,5 +280,66 @@ float unsmoothedBleu(const std::vector<float>& stats) { return exp(logbleu); } +vector<float> BleuScorer::ScoreNbestList(string scoreFile, string featureFile) { + vector<string> scoreFiles; + vector<string> featureFiles; + scoreFiles.push_back(scoreFile); + featureFiles.push_back(featureFile); + + vector<FeatureDataIterator> featureDataIters; + vector<ScoreDataIterator> scoreDataIters; + for (size_t i = 0; i < featureFiles.size(); ++i) { + featureDataIters.push_back(FeatureDataIterator(featureFiles[i])); + scoreDataIters.push_back(ScoreDataIterator(scoreFiles[i])); + } + + vector<pair<size_t,size_t> > hypotheses; + if (featureDataIters[0] == FeatureDataIterator::end()) { + cerr << "Error: at the end of feature data iterator" << endl; + exit(1); + } + for (size_t i = 0; i < featureFiles.size(); ++i) { + if (featureDataIters[i] == FeatureDataIterator::end()) { + cerr << "Error: Feature file " << i << " ended prematurely" << endl; + exit(1); + } + if (scoreDataIters[i] == ScoreDataIterator::end()) { + cerr << "Error: Score file " << i << " ended prematurely" << endl; + exit(1); + } + if (featureDataIters[i]->size() != scoreDataIters[i]->size()) { + cerr << "Error: features and scores have different size" << endl; + exit(1); + } + for (size_t j = 0; j < featureDataIters[i]->size(); ++j) { + hypotheses.push_back(pair<size_t,size_t>(i,j)); + } + } + + // score the nbest list + vector<float> bleuScores; + for (size_t i=0; i < hypotheses.size(); ++i) { + pair<size_t,size_t> translation = hypotheses[i]; + float bleu = sentenceLevelBleuPlusOne(scoreDataIters[translation.first]->operator[](translation.second)); + bleuScores.push_back(bleu); + } + return bleuScores; +} + +float BleuScorer::sentenceLevelBleuPlusOne(const vector<float>& stats) { + float logbleu = 0.0; + const unsigned int bleu_order = 4; + for (unsigned int j=0; j<bleu_order; j++) { + //cerr << (stats.get(2*j)+1) << "/" << (stats.get(2*j+1)+1) << " "; + logbleu += log(stats[2*j]+1) - log(stats[2*j+1]+1); + } + logbleu /= bleu_order; + float brevity = 1.0 - (float)stats[(bleu_order*2)]/stats[1]; + if (brevity < 0.0) { + logbleu += brevity; + } + //cerr << brevity << " -> " << exp(logbleu) << endl; + return exp(logbleu); } +} diff --git a/mert/BleuScorer.h b/mert/BleuScorer.h index 38669b740..1927183a9 100644 --- a/mert/BleuScorer.h +++ b/mert/BleuScorer.h @@ -18,6 +18,8 @@ const int kBleuNgramOrder = 4; class NgramCounts; class Reference; +using namespace std; + /** * Bleu scoring */ @@ -32,6 +34,9 @@ public: explicit BleuScorer(const std::string& config = ""); ~BleuScorer(); + + static vector<float> ScoreNbestList(string scoreFile, string featureFile); + static float sentenceLevelBleuPlusOne(const vector<float>& stats); virtual void setReferenceFiles(const std::vector<std::string>& referenceFiles); virtual void prepareStats(std::size_t sid, const std::string& text, ScoreStats& entry); diff --git a/mert/Data.cpp b/mert/Data.cpp index 78d6bda86..3c0a03db0 100644 --- a/mert/Data.cpp +++ b/mert/Data.cpp @@ -21,25 +21,24 @@ using namespace std; namespace MosesTuning { - -Data::Data() - : m_scorer(NULL), - m_num_scores(0), - m_sparse_flag(false), - m_score_data(), - m_feature_data() {} - -Data::Data(Scorer* scorer) +Data::Data(Scorer* scorer, const string& sparse_weights_file) : m_scorer(scorer), m_score_type(m_scorer->getName()), m_num_scores(0), - m_sparse_flag(false), m_score_data(new ScoreData(m_scorer)), m_feature_data(new FeatureData) { TRACE_ERR("Data::m_score_type " << m_score_type << endl); TRACE_ERR("Data::Scorer type from Scorer: " << m_scorer->getName() << endl); + if (sparse_weights_file.size()) { + m_sparse_weights.load(sparse_weights_file); + ostringstream msg; + msg << "Data::sparse_weights {"; + m_sparse_weights.write(msg,"="); + msg << "}"; + TRACE_ERR(msg.str() << std::endl); + } } //ADDED BY TS @@ -126,10 +125,8 @@ void Data::removeDuplicates() { //END_ADDED void Data::load(const std::string &featfile, const std::string &scorefile) { - m_feature_data->load(featfile); + m_feature_data->load(featfile, m_sparse_weights); m_score_data->load(scorefile); - if (m_feature_data->hasSparseFeatures()) - m_sparse_flag = true; } void Data::loadNBest(const string &file) @@ -236,18 +233,11 @@ void Data::AddFeatures(const string& str, string name = substr; getNextPound(buf, substr); feature_entry.addSparse(name, atof(substr.c_str())); - m_sparse_flag = true; } } m_feature_data->add(feature_entry, sentence_index); } -// TODO -void Data::mergeSparseFeatures() { - cerr << "ERROR: sparse features can only be trained with pairwise ranked optimizer (PRO), not traditional MERT\n"; - exit(1); -} - void Data::createShards(size_t shard_count, float shard_size, const string& scorerconfig, vector<Data>& shards) { @@ -282,7 +272,6 @@ void Data::createShards(size_t shard_count, float shard_size, const string& scor shards.push_back(Data(scorer)); shards.back().m_score_type = m_score_type; shards.back().m_num_scores = m_num_scores; - shards.back().m_sparse_flag = m_sparse_flag; for (size_t i = 0; i < shard_contents.size(); ++i) { shards.back().m_feature_data->add(m_feature_data->get(shard_contents[i])); shards.back().m_score_data->add(m_score_data->get(shard_contents[i])); diff --git a/mert/Data.h b/mert/Data.h index aa69f4edf..528a396cd 100644 --- a/mert/Data.h +++ b/mert/Data.h @@ -32,13 +32,12 @@ private: Scorer* m_scorer; std::string m_score_type; std::size_t m_num_scores; - bool m_sparse_flag; ScoreDataHandle m_score_data; FeatureDataHandle m_feature_data; + SparseVector m_sparse_weights; public: - explicit Data(Scorer* scorer); - Data(); + explicit Data(Scorer* scorer, const std::string& sparseweightsfile=""); void clear() { m_score_data->clear(); @@ -55,14 +54,9 @@ public: return m_feature_data->NumberOfFeatures(); } - void NumberOfFeatures(std::size_t v) { m_feature_data->NumberOfFeatures(v); } - std::string Features() const { return m_feature_data->Features(); } void Features(const std::string &f) { m_feature_data->Features(f); } - bool hasSparseFeatures() const { return m_sparse_flag; } - void mergeSparseFeatures(); - void loadNBest(const std::string &file); void load(const std::string &featfile, const std::string &scorefile); diff --git a/mert/FeatureArray.cpp b/mert/FeatureArray.cpp index 0f297e8cb..0e3114bca 100644 --- a/mert/FeatureArray.cpp +++ b/mert/FeatureArray.cpp @@ -19,7 +19,7 @@ namespace MosesTuning FeatureArray::FeatureArray() - : m_index(""), m_num_features(0), m_sparse_flag(false) {} + : m_index(""), m_num_features(0){} FeatureArray::~FeatureArray() {} @@ -81,19 +81,17 @@ void FeatureArray::loadbin(istream* is, size_t n) } } -void FeatureArray::loadtxt(istream* is, size_t n) +void FeatureArray::loadtxt(istream* is, const SparseVector& sparseWeights, size_t n) { FeatureStats entry(m_num_features); - for (size_t i = 0; i < n; i++) { - entry.loadtxt(is); + for (size_t i=0 ; i < n; i++) { + entry.loadtxt(is, sparseWeights); add(entry); - if (entry.getSparse().size()>0) - m_sparse_flag = true; } } -void FeatureArray::load(istream* is) +void FeatureArray::load(istream* is, const SparseVector& sparseWeights) { size_t number_of_entries = 0; bool binmode = false; @@ -128,7 +126,7 @@ void FeatureArray::load(istream* is) if (binmode) { loadbin(is, number_of_entries); } else { - loadtxt(is, number_of_entries); + loadtxt(is, sparseWeights, number_of_entries); } getline(*is, stringBuf); @@ -141,15 +139,6 @@ void FeatureArray::load(istream* is) } } -void FeatureArray::load(const string &file) -{ - TRACE_ERR("loading data from " << file << endl); - inputfilestream input_stream(file); // matches a stream with a file. Opens the file - istream* is = &input_stream; - load(is); - input_stream.close(); -} - void FeatureArray::merge(FeatureArray& e) { //dummy implementation diff --git a/mert/FeatureArray.h b/mert/FeatureArray.h index bac743fe1..d50be0fa0 100644 --- a/mert/FeatureArray.h +++ b/mert/FeatureArray.h @@ -27,11 +27,11 @@ class FeatureArray private: // idx to identify the utterance. It can differ from // the index inside the vector. + std::string m_index; featarray_t m_array; std::size_t m_num_features; std::string m_features; - bool m_sparse_flag; public: FeatureArray(); @@ -39,7 +39,6 @@ public: void clear() { m_array.clear(); } - bool hasSparseFeatures() const { return m_sparse_flag; } std::string getIndex() const { return m_index; } void setIndex(const std::string& value) { m_index = value; } @@ -75,10 +74,9 @@ public: void save(const std::string &file, bool bin=false); void save(bool bin=false); - void loadtxt(std::istream* is, std::size_t n); + void loadtxt(std::istream* is, const SparseVector& sparseWeights, std::size_t n); void loadbin(std::istream* is, std::size_t n); - void load(std::istream* is); - void load(const std::string &file); + void load(std::istream* is, const SparseVector& sparseWeights); bool check_consistency() const; }; diff --git a/mert/FeatureData.cpp b/mert/FeatureData.cpp index 8daec204a..e8c6d79ca 100644 --- a/mert/FeatureData.cpp +++ b/mert/FeatureData.cpp @@ -18,12 +18,9 @@ namespace MosesTuning { -static const float MIN_FLOAT = -1.0 * numeric_limits<float>::max(); -static const float MAX_FLOAT = numeric_limits<float>::max(); FeatureData::FeatureData() - : m_num_features(0), - m_sparse_flag(false) {} + : m_num_features(0) {} void FeatureData::save(ostream* os, bool bin) { @@ -45,7 +42,7 @@ void FeatureData::save(bool bin) { save(&cout, bin); } -void FeatureData::load(istream* is) +void FeatureData::load(istream* is, const SparseVector& sparseWeights) { FeatureArray entry; @@ -56,7 +53,7 @@ void FeatureData::load(istream* is) } entry.clear(); - entry.load(is); + entry.load(is, sparseWeights); if (entry.size() == 0) break; @@ -64,15 +61,12 @@ void FeatureData::load(istream* is) if (size() == 0) setFeatureMap(entry.Features()); - if (entry.hasSparseFeatures()) - m_sparse_flag = true; - add(entry); } } -void FeatureData::load(const string &file) +void FeatureData::load(const string &file, const SparseVector& sparseWeights) { TRACE_ERR("loading feature data from " << file << endl); inputfilestream input_stream(file); // matches a stream with a file. Opens the file @@ -80,7 +74,7 @@ void FeatureData::load(const string &file) throw runtime_error("Unable to open feature file: " + file); } istream* is = &input_stream; - load(is); + load(is, sparseWeights); input_stream.close(); } @@ -157,13 +151,7 @@ string FeatureData::ToString() const { { stringstream ss; ss << "number of features: " << m_num_features - << ", features: " << m_features - << ", sparse flag: "; - if (m_sparse_flag) { - ss << "yes, "; - } else { - ss << "no, "; - } + << ", features: " << m_features; res.append(ss.str()); } diff --git a/mert/FeatureData.h b/mert/FeatureData.h index 5bd0f5d6f..4e33dd3c3 100644 --- a/mert/FeatureData.h +++ b/mert/FeatureData.h @@ -23,7 +23,6 @@ class FeatureData private: std::size_t m_num_features; std::string m_features; - bool m_sparse_flag; std::map<std::string, std::size_t> m_feature_name_to_index; // map from name to index of features std::map<std::size_t, std::string> m_index_to_feature_name; // map from index to name of features featdata_t m_array; @@ -36,14 +35,15 @@ public: void clear() { m_array.clear(); } - bool hasSparseFeatures() const { return m_sparse_flag; } - FeatureArray get(const std::string& idx) { return m_array.at(getIndex(idx)); } - - FeatureArray& get(std::size_t idx) { return m_array.at(idx); } - const FeatureArray& get(std::size_t idx) const { return m_array.at(idx); } + FeatureArray& get(size_t idx) { + return m_array.at(idx); + } + const FeatureArray& get(size_t idx) const { + return m_array.at(idx); + } inline bool exists(const std::string& sent_idx) const { return exists(getIndex(sent_idx)); @@ -76,8 +76,8 @@ public: void save(std::ostream* os, bool bin=false); void save(bool bin=false); - void load(std::istream* is); - void load(const std::string &file); + void load(std::istream* is, const SparseVector& sparseWeights); + void load(const std::string &file, const SparseVector& sparseWeights); bool check_consistency() const; diff --git a/mert/FeatureDataIterator.cpp b/mert/FeatureDataIterator.cpp index f9acbb64c..7ada05a61 100644 --- a/mert/FeatureDataIterator.cpp +++ b/mert/FeatureDataIterator.cpp @@ -86,7 +86,7 @@ void FeatureDataIterator::readNext() { StringPiece line = m_in->ReadLine(); m_next.push_back(FeatureDataItem()); for (TokenIter<AnyCharacter, true> token(line, AnyCharacter(" \t")); token; ++token) { - TokenIter<AnyCharacter,false> value(*token,AnyCharacter(":")); + TokenIter<AnyCharacterLast,false> value(*token,AnyCharacterLast(":")); if (!value) throw FileFormatException(m_in->FileName(), line.as_string()); StringPiece first = *value; ++value; diff --git a/mert/FeatureStats.cpp b/mert/FeatureStats.cpp index 6f80d5f89..22f62e234 100644 --- a/mert/FeatureStats.cpp +++ b/mert/FeatureStats.cpp @@ -10,6 +10,8 @@ #include <fstream> #include <cmath> +#include <stdexcept> + #include <boost/functional/hash.hpp> #include "Util.h" @@ -65,28 +67,55 @@ void SparseVector::clear() { m_fvector.clear(); } -SparseVector& SparseVector::operator-=(const SparseVector& rhs) { - //All the elements that have values in *this - for (fvector_t::iterator i = m_fvector.begin(); i != m_fvector.end(); ++i) { - m_fvector[i->first] = i->second - rhs.get(i->first); +void SparseVector::load(const string& file) { + ifstream in(file.c_str()); + if (!in) { + throw runtime_error("Failed to open sparse weights file: " + file); + } + string line; + while(getline(in,line)) { + if (line[0] == '#') continue; + istringstream linestream(line); + string name; + float value; + linestream >> name; + linestream >> value; + set(name,value); } +} + +SparseVector& SparseVector::operator-=(const SparseVector& rhs) { - //Any elements in rhs, that have no value in *this for (fvector_t::const_iterator i = rhs.m_fvector.begin(); i != rhs.m_fvector.end(); ++i) { - if (m_fvector.find(i->first) == m_fvector.end()) { - m_fvector[i->first] = -(i->second); - } + m_fvector[i->first] = get(i->first) - (i->second); } return *this; } +FeatureStatsType SparseVector::inner_product(const SparseVector& rhs) const { + FeatureStatsType product = 0.0; + for (fvector_t::const_iterator i = m_fvector.begin(); + i != m_fvector.end(); ++i) { + product += ((i->second) * (rhs.get(i->first))); + } + return product; +} + SparseVector operator-(const SparseVector& lhs, const SparseVector& rhs) { SparseVector res(lhs); res -= rhs; return res; } +FeatureStatsType inner_product(const SparseVector& lhs, const SparseVector& rhs) { + if (lhs.size() >= rhs.size()) { + return rhs.inner_product(lhs); + } else { + return lhs.inner_product(rhs); + } +} + std::vector<std::size_t> SparseVector::feats() const { std::vector<std::size_t> toRet; for(fvector_t::const_iterator iter = m_fvector.begin(); @@ -123,7 +152,6 @@ std::size_t hash_value(SparseVector const& item) { return hasher(item.m_fvector); } - FeatureStats::FeatureStats() : m_available_size(kAvailableSize), m_entries(0), m_array(new FeatureStatsType[m_available_size]) {} @@ -135,12 +163,6 @@ FeatureStats::FeatureStats(const size_t size) memset(m_array, 0, GetArraySizeWithBytes()); } -FeatureStats::FeatureStats(string &theString) - : m_available_size(0), m_entries(0), m_array(NULL) -{ - set(theString); -} - FeatureStats::~FeatureStats() { if (m_array) { @@ -190,7 +212,7 @@ void FeatureStats::addSparse(const string& name, FeatureStatsType v) m_map.set(name,v); } -void FeatureStats::set(string &theString) +void FeatureStats::set(string &theString, const SparseVector& sparseWeights ) { string substring, stringBuf; reset(); @@ -207,6 +229,26 @@ void FeatureStats::set(string &theString) addSparse(substring.substr(0,separator), atof(substring.substr(separator+1).c_str()) ); } } + + if (sparseWeights.size()) { + //Merge the sparse features + FeatureStatsType merged = inner_product(sparseWeights, m_map); + add(merged); + /* + cerr << "Merged "; + sparseWeights.write(cerr,"="); + cerr << " and "; + map_.write(cerr,"="); + cerr << " to give " << merged << endl; + */ + m_map.clear(); + } + /* + cerr << "FS: "; + for (size_t i = 0; i < entries_; ++i) { + cerr << array_[i] << " "; + } + cerr << endl;*/ } void FeatureStats::loadbin(istream* is) @@ -215,22 +257,11 @@ void FeatureStats::loadbin(istream* is) static_cast<streamsize>(GetArraySizeWithBytes())); } -void FeatureStats::loadtxt(istream* is) +void FeatureStats::loadtxt(istream* is, const SparseVector& sparseWeights) { string line; getline(*is, line); - set(line); -} - -void FeatureStats::loadtxt(const string &file) -{ - ifstream ifs(file.c_str(), ios::in); - if (!ifs) { - cerr << "Failed to open " << file << endl; - exit(1); - } - istream* is = &ifs; - loadtxt(is); + set(line, sparseWeights); } void FeatureStats::savetxt(const string &file) diff --git a/mert/FeatureStats.h b/mert/FeatureStats.h index 9085fb7dc..883a89b97 100644 --- a/mert/FeatureStats.h +++ b/mert/FeatureStats.h @@ -31,11 +31,13 @@ public: FeatureStatsType get(std::size_t id) const; void set(const std::string& name, FeatureStatsType value); void clear(); + void load(const std::string& file); std::size_t size() const { return m_fvector.size(); } void write(std::ostream& out, const std::string& sep = " ") const; SparseVector& operator-=(const SparseVector& rhs); + FeatureStatsType inner_product(const SparseVector& rhs) const; // Added by cherryc std::vector<std::size_t> feats() const; @@ -52,6 +54,7 @@ private: }; SparseVector operator-(const SparseVector& lhs, const SparseVector& rhs); +FeatureStatsType inner_product(const SparseVector& lhs, const SparseVector& rhs); class FeatureStats { @@ -66,7 +69,6 @@ private: public: FeatureStats(); explicit FeatureStats(const std::size_t size); - explicit FeatureStats(std::string &theString); ~FeatureStats(); @@ -97,7 +99,7 @@ public: const SparseVector& getSparse() const { return m_map; } - void set(std::string &theString); + void set(std::string &theString, const SparseVector& sparseWeights); inline std::size_t bytes() const { return GetArraySizeWithBytes(); } @@ -114,8 +116,7 @@ public: void savebin(std::ostream* os); void savetxt(); - void loadtxt(const std::string &file); - void loadtxt(std::istream* is); + void loadtxt(std::istream* is, const SparseVector& sparseWeights); void loadbin(std::istream* is); /** diff --git a/mert/Jamfile b/mert/Jamfile index 84ff03701..828d5c367 100644 --- a/mert/Jamfile +++ b/mert/Jamfile @@ -51,13 +51,13 @@ PermutationScorer.cpp StatisticsBasedScorer.cpp ../util//kenutil m ..//z ; -exe mert : mert.cpp mert_lib ../moses/src//ThreadPool ; +exe mert : mert.cpp mert_lib bleu_lib ../moses/src//ThreadPool ; -exe extractor : extractor.cpp mert_lib ; +exe extractor : extractor.cpp mert_lib bleu_lib ; -exe evaluator : evaluator.cpp mert_lib ; +exe evaluator : evaluator.cpp mert_lib bleu_lib ; -exe pro : pro.cpp mert_lib ..//boost_program_options ; +exe pro : pro.cpp mert_lib bleu_lib ..//boost_program_options ; exe kbmira : kbmira.cpp mert_lib ..//boost_program_options ; @@ -75,3 +75,7 @@ unit-test timer_test : TimerTest.cpp mert_lib ..//boost_unit_test_framework ; unit-test util_test : UtilTest.cpp mert_lib ..//boost_unit_test_framework ; unit-test vocabulary_test : VocabularyTest.cpp mert_lib ..//boost_unit_test_framework ; +install legacy : programs : <location>. ; + +lib bleu_lib : BleuScorer.cpp mert_lib : : : <include>. ; + diff --git a/mert/extractor.cpp b/mert/extractor.cpp index d75455b0e..077d9b94c 100644 --- a/mert/extractor.cpp +++ b/mert/extractor.cpp @@ -212,7 +212,7 @@ int main(int argc, char** argv) if (referenceFiles.size() > 0) scorer->setReferenceFiles(referenceFiles); - PrintUserTime("References loaded"); +// PrintUserTime("References loaded"); Data data(scorer.get()); @@ -221,14 +221,14 @@ int main(int argc, char** argv) data.load(prevFeatureDataFiles.at(i), prevScoreDataFiles.at(i)); } - PrintUserTime("Previous data loaded"); +// PrintUserTime("Previous data loaded"); // computing score statistics of each nbest file for (size_t i = 0; i < nbestFiles.size(); i++) { data.loadNBest(nbestFiles.at(i)); } - PrintUserTime("Nbest entries loaded and scored"); +// PrintUserTime("Nbest entries loaded and scored"); //ADDED_BY_TS if (!option.allowDuplicates) { diff --git a/mert/mert.cpp b/mert/mert.cpp index 5f43e2b16..4352927dc 100644 --- a/mert/mert.cpp +++ b/mert/mert.cpp @@ -38,6 +38,7 @@ const char kDefaultScorerFile[] = "statscore.data"; const char kDefaultFeatureFile[] = "features.data"; const char kDefaultInitFile[] = "init.opt"; const char kDefaultPositiveString[] = ""; +const char kDefaultSparseWeightsFile[] = ""; // Used when saving optimized weights. const char kOutputFile[] = "weights.txt"; @@ -99,49 +100,48 @@ bool WriteFinalWeights(const char* filename, const Point& point) { void usage(int ret) { - cerr << "usage: mert -d <dimensions> (mandatory)" << endl; - cerr << "[-n] retry ntimes (default 1)" << endl; - cerr << "[-m] number of random directions in powell (default 0)"<< endl; - cerr << "[-o] the indexes to optimize(default all)" << endl; - cerr << "[-t] the optimizer(default " << kDefaultOptimizer << ")" << endl; - cerr << "[-r] the random seed (defaults to system clock)" << endl; - cerr << "[--sctype|-s] the scorer type (default " << kDefaultScorer << ")" << endl; - cerr << "[--scconfig|-c] configuration string passed to scorer" << endl; - cerr << "[--scfile|-S] comma separated list of scorer data files (default " << kDefaultScorerFile << ")" << endl; - cerr << "[--ffile|-F] comma separated list of feature data files (default " << kDefaultFeatureFile << ")" << endl; - cerr << "[--ifile|-i] the starting point data file (default " << kDefaultInitFile << ")" << endl; - cerr << "[--positive|-P] indexes with positive weights (default none)"<<endl; + cerr<<"usage: mert -d <dimensions> (mandatory )"<<endl; + cerr<<"[-n] retry ntimes (default 1)"<<endl; + cerr<<"[-m] number of random directions in powell (default 0)"<<endl; + cerr<<"[-o] the indexes to optimize(default all)"<<endl; + cerr<<"[-t] the optimizer(default powell)"<<endl; + cerr<<"[-r] the random seed (defaults to system clock)"<<endl; + cerr<<"[--sctype|-s] the scorer type (default BLEU)"<<endl; + cerr<<"[--scconfig|-c] configuration string passed to scorer"<<endl; + cerr<<"[--scfile|-S] comma separated list of scorer data files (default score.data)"<<endl; + cerr<<"[--ffile|-F] comma separated list of feature data files (default feature.data)"<<endl; + cerr<<"[--ifile|-i] the starting point data file (default init.opt)"<<endl; + cerr<<"[--sparse-weights|-p] required for merging sparse features"<<endl; #ifdef WITH_THREADS - cerr << "[--threads|-T] use multiple threads (default 1)" << endl; + cerr<<"[--threads|-T] use multiple threads (default 1)"<<endl; #endif - cerr << "[--shard-count] Split data into shards, optimize for each shard and average" << endl; - cerr << "[--shard-size] Shard size as proportion of data. If 0, use non-overlapping shards" << endl; - cerr << "[-v] verbose level" << endl; - cerr << "[--help|-h] print this message and exit" << endl; + cerr<<"[--shard-count] Split data into shards, optimize for each shard and average"<<endl; + cerr<<"[--shard-size] Shard size as proportion of data. If 0, use non-overlapping shards"<<endl; + cerr<<"[-v] verbose level"<<endl; + cerr<<"[--help|-h] print this message and exit"<<endl; exit(ret); } static struct option long_options[] = { {"pdim", 1, 0, 'd'}, - {"ntry", 1, 0, 'n'}, - {"nrandom", 1, 0, 'm'}, - {"rseed", required_argument, 0, 'r'}, - {"optimize", 1, 0, 'o'}, - {"pro", required_argument, 0, 'p'}, - {"positive",1,0,'P'}, - {"type", 1, 0, 't'}, - {"sctype", 1, 0, 's'}, - {"scconfig", required_argument, 0, 'c'}, - {"scfile", 1, 0, 'S'}, - {"ffile", 1, 0, 'F'}, - {"ifile", 1, 0, 'i'}, + {"ntry",1,0,'n'}, + {"nrandom",1,0,'m'}, + {"rseed",required_argument,0,'r'}, + {"optimize",1,0,'o'}, + {"type",1,0,'t'}, + {"sctype",1,0,'s'}, + {"scconfig",required_argument,0,'c'}, + {"scfile",1,0,'S'}, + {"ffile",1,0,'F'}, + {"ifile",1,0,'i'}, + {"sparse-weights",required_argument,0,'p'}, #ifdef WITH_THREADS - {"threads", required_argument, 0, 'T'}, + {"threads", required_argument,0,'T'}, #endif {"shard-count", required_argument, 0, 'a'}, {"shard-size", required_argument, 0, 'b'}, - {"verbose", 1, 0, 'v'}, - {"help", no_argument, 0, 'h'}, + {"verbose",1,0,'v'}, + {"help",no_argument,0,'h'}, {0, 0, 0, 0} }; @@ -159,6 +159,7 @@ struct ProgramOption { string feature_file; string init_file; string positive_string; + string sparse_weights_file; size_t num_threads; float shard_size; size_t shard_count; @@ -177,6 +178,7 @@ struct ProgramOption { feature_file(kDefaultFeatureFile), init_file(kDefaultInitFile), positive_string(kDefaultPositiveString), + sparse_weights_file(kDefaultSparseWeightsFile), num_threads(1), shard_size(0), shard_count(0) { } @@ -222,6 +224,9 @@ void ParseCommandOptions(int argc, char** argv, ProgramOption* opt) { case 'i': opt->init_file = string(optarg); break; + case 'p': + opt->sparse_weights_file=string(optarg); + break; case 'v': setverboselevel(strtol(optarg, NULL, 10)); break; @@ -286,6 +291,8 @@ int main(int argc, char **argv) srandom(time(NULL)); } + if (option.sparse_weights_file.size()) ++option.pdim; + // read in starting points string onefile; while (!option.init_file.empty()) { @@ -349,7 +356,7 @@ int main(int argc, char **argv) ScorerFactory::getScorer(option.scorer_type, option.scorer_config)); //load data - Data data(scorer.get()); + Data data(scorer.get(), option.sparse_weights_file); for (size_t i = 0; i < ScoreDataFiles.size(); i++) { cerr<<"Loading Data from: "<< ScoreDataFiles.at(i) << " and " << FeatureDataFiles.at(i) << endl; @@ -419,12 +426,6 @@ int main(int argc, char **argv) } } - // treat sparse features just like regular features - if (data.hasSparseFeatures()) { - data.mergeSparseFeatures(); - } - - #ifdef WITH_THREADS cerr << "Creating a pool of " << option.num_threads << " threads" << endl; Moses::ThreadPool pool(option.num_threads); diff --git a/mert/pro.cpp b/mert/pro.cpp index 84ce09a8e..8055b19bd 100644 --- a/mert/pro.cpp +++ b/mert/pro.cpp @@ -41,6 +41,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "BleuScorer.h" #include "FeatureDataIterator.h" #include "ScoreDataIterator.h" +#include "BleuScorer.h" using namespace std; using namespace MosesTuning; diff --git a/mira/Decoder.cpp b/mira/Decoder.cpp new file mode 100644 index 000000000..f75c2613a --- /dev/null +++ b/mira/Decoder.cpp @@ -0,0 +1,394 @@ +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2009 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#include "Decoder.h" +#include "Manager.h" +#include "ChartManager.h" +#include "Sentence.h" +#include "InputType.h" +#include "TranslationSystem.h" +#include "Phrase.h" +#include "TrellisPathList.h" +#include "ChartTrellisPathList.h" +#include "ChartTrellisPath.h" +#include "IOWrapper.h" + +using namespace std; +using namespace Moses; + + +namespace Mira { + + /** + * Allocates a char* and copies string into it. + **/ + static char* strToChar(const string& s) { + char* c = new char[s.size()+1]; + strcpy(c,s.c_str()); + return c; + } + + MosesDecoder::MosesDecoder(const string& inifile, int debuglevel, int argc, vector<string> decoder_params) + : m_manager(NULL) { + static int BASE_ARGC = 8; + Parameter* params = new Parameter(); + char ** mosesargv = new char*[BASE_ARGC + argc]; + mosesargv[0] = strToChar("-f"); + mosesargv[1] = strToChar(inifile); + mosesargv[2] = strToChar("-v"); + stringstream dbgin; + dbgin << debuglevel; + mosesargv[3] = strToChar(dbgin.str()); + mosesargv[4] = strToChar("-use-persistent-cache"); + mosesargv[5] = strToChar("0"); + mosesargv[6] = strToChar("-persistent-cache-size"); + mosesargv[7] = strToChar("0"); + + for (int i = 0; i < argc; ++i) { + char *cstr = &(decoder_params[i])[0]; + mosesargv[BASE_ARGC + i] = cstr; + } + + if (!params->LoadParam(BASE_ARGC + argc,mosesargv)) { + cerr << "Loading static data failed, exit." << endl; + exit(1); + } + StaticData::LoadDataStatic(params, "mira"); + for (int i = 0; i < BASE_ARGC; ++i) { + delete[] mosesargv[i]; + } + delete[] mosesargv; + + const StaticData &staticData = StaticData::Instance(); + m_bleuScoreFeature = staticData.GetBleuScoreFeature(); + } + + void MosesDecoder::cleanup(bool chartDecoding) { + delete m_manager; + if (chartDecoding) + delete m_chartManager; + else + delete m_sentence; + } + + vector< vector<const Word*> > MosesDecoder::getNBest(const std::string& source, + size_t sentenceid, + size_t nBestSize, + float bleuObjectiveWeight, + float bleuScoreWeight, + vector< ScoreComponentCollection>& featureValues, + vector< float>& bleuScores, + vector< float>& modelScores, + size_t numReturnedTranslations, + bool realBleu, + bool distinct, + bool avgRefLength, + size_t rank, + size_t epoch, + string filename) + { + StaticData &staticData = StaticData::InstanceNonConst(); + bool chartDecoding = (staticData.GetSearchAlgorithm() == ChartDecoding); + initialize(staticData, source, sentenceid, bleuObjectiveWeight, bleuScoreWeight, avgRefLength, chartDecoding); + const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT); + + // run the decoder + if (chartDecoding) { + return runChartDecoder(source, sentenceid, nBestSize, bleuObjectiveWeight, bleuScoreWeight, + featureValues, bleuScores, modelScores, numReturnedTranslations, realBleu, distinct, rank, epoch, + system); + } + else { + SearchAlgorithm search = staticData.GetSearchAlgorithm(); + return runDecoder(source, sentenceid, nBestSize, bleuObjectiveWeight, bleuScoreWeight, + featureValues, bleuScores, modelScores, numReturnedTranslations, realBleu, distinct, rank, epoch, + search, system, filename); + } + } + + vector< vector<const Word*> > MosesDecoder::runDecoder(const std::string& source, + size_t sentenceid, + size_t nBestSize, + float bleuObjectiveWeight, + float bleuScoreWeight, + vector< ScoreComponentCollection>& featureValues, + vector< float>& bleuScores, + vector< float>& modelScores, + size_t numReturnedTranslations, + bool realBleu, + bool distinct, + size_t rank, + size_t epoch, + SearchAlgorithm& search, + const TranslationSystem& system, + string filename) { + // run the decoder + m_manager = new Moses::Manager(0,*m_sentence, search, &system); + m_manager->ProcessSentence(); + TrellisPathList nBestList; + m_manager->CalcNBest(nBestSize, nBestList, distinct); + + // optionally print nbest to file (to extract scores and features.. currently just for sentence bleu scoring) + if (filename != "") { + ofstream out(filename.c_str()); + if (!out) { + ostringstream msg; + msg << "Unable to open " << filename; + throw runtime_error(msg.str()); + } + // TODO: handle sentence id (for now always 0) + //OutputNBest(out, nBestList, StaticData::Instance().GetOutputFactorOrder(),m_manager->GetTranslationSystem(), 0, false); + out.close(); + } + + // read off the feature values and bleu scores for each sentence in the nbest list + Moses::TrellisPathList::const_iterator iter; + for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) { + const Moses::TrellisPath &path = **iter; + featureValues.push_back(path.GetScoreBreakdown()); + float bleuScore, dynBleuScore, realBleuScore; + if (realBleu) realBleuScore = m_bleuScoreFeature->CalculateBleu(path.GetTargetPhrase()); + else dynBleuScore = getBleuScore(featureValues.back()); + bleuScore = realBleu ? realBleuScore : dynBleuScore; + bleuScores.push_back(bleuScore); + + //std::cout << "Score breakdown: " << path.GetScoreBreakdown() << endl; + float scoreWithoutBleu = path.GetTotalScore() - (bleuObjectiveWeight * bleuScoreWeight * bleuScore); + modelScores.push_back(scoreWithoutBleu); + + if (iter != nBestList.begin()) + cerr << endl; + cerr << "Rank " << rank << ", epoch " << epoch << ", \"" << path.GetTargetPhrase() << "\", score: " + << scoreWithoutBleu << ", Bleu: " << bleuScore << ", total: " << path.GetTotalScore(); + if (m_bleuScoreFeature->Enabled() && realBleu) + cerr << " (d-bleu: " << dynBleuScore << ", r-bleu: " << realBleuScore << ") "; + + // set bleu score to zero in the feature vector since we do not want to optimise its weight + setBleuScore(featureValues.back(), 0); + } + + // prepare translations to return + vector< vector<const Word*> > translations; + for (size_t i=0; i < numReturnedTranslations && i < nBestList.GetSize(); ++i) { + const TrellisPath &path = nBestList.at(i); + Phrase phrase = path.GetTargetPhrase(); + + vector<const Word*> translation; + for (size_t pos = 0; pos < phrase.GetSize(); ++pos) { + const Word &word = phrase.GetWord(pos); + Word *newWord = new Word(word); + translation.push_back(newWord); + } + translations.push_back(translation); + } + + return translations; + } + + vector< vector<const Word*> > MosesDecoder::runChartDecoder(const std::string& source, + size_t sentenceid, + size_t nBestSize, + float bleuObjectiveWeight, + float bleuScoreWeight, + vector< ScoreComponentCollection>& featureValues, + vector< float>& bleuScores, + vector< float>& modelScores, + size_t numReturnedTranslations, + bool realBleu, + bool distinct, + size_t rank, + size_t epoch, + const TranslationSystem& system) { + // run the decoder + m_chartManager = new ChartManager(*m_sentence, &system); + m_chartManager->ProcessSentence(); + ChartTrellisPathList nBestList; + m_chartManager->CalcNBest(nBestSize, nBestList, distinct); + + // read off the feature values and bleu scores for each sentence in the nbest list + ChartTrellisPathList::const_iterator iter; + for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) { + const Moses::ChartTrellisPath &path = **iter; + featureValues.push_back(path.GetScoreBreakdown()); + float bleuScore, dynBleuScore, realBleuScore; + dynBleuScore = getBleuScore(featureValues.back()); + realBleuScore = m_bleuScoreFeature->CalculateBleu(path.GetOutputPhrase()); + bleuScore = realBleu ? realBleuScore : dynBleuScore; + bleuScores.push_back(bleuScore); + + //std::cout << "Score breakdown: " << path.GetScoreBreakdown() << endl; + float scoreWithoutBleu = path.GetTotalScore() - (bleuObjectiveWeight * bleuScoreWeight * bleuScore); + modelScores.push_back(scoreWithoutBleu); + + if (iter != nBestList.begin()) + cerr << endl; + cerr << "Rank " << rank << ", epoch " << epoch << ", \"" << path.GetOutputPhrase() << "\", score: " + << scoreWithoutBleu << ", Bleu: " << bleuScore << ", total: " << path.GetTotalScore(); + if (m_bleuScoreFeature->Enabled() && realBleu) + cerr << " (d-bleu: " << dynBleuScore << ", r-bleu: " << realBleuScore << ") "; + + // set bleu score to zero in the feature vector since we do not want to optimise its weight + setBleuScore(featureValues.back(), 0); + } + + // prepare translations to return + vector< vector<const Word*> > translations; + for (iter = nBestList.begin() ; iter != nBestList.end() ; ++iter) { + const ChartTrellisPath &path = **iter; + Phrase phrase = path.GetOutputPhrase(); + + vector<const Word*> translation; + for (size_t pos = 0; pos < phrase.GetSize(); ++pos) { + const Word &word = phrase.GetWord(pos); + Word *newWord = new Word(word); + translation.push_back(newWord); + } + translations.push_back(translation); + } + + return translations; + } + + void MosesDecoder::outputNBestList(const std::string& source, size_t sentenceid, + size_t nBestSize, float bleuObjectiveWeight, float bleuScoreWeight, + bool distinctNbest, bool avgRefLength, string filename, ofstream& streamOut) { + StaticData &staticData = StaticData::InstanceNonConst(); + bool chartDecoding = (staticData.GetSearchAlgorithm() == ChartDecoding); + initialize(staticData, source, sentenceid, bleuObjectiveWeight, bleuScoreWeight, avgRefLength, chartDecoding); + const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT); + + if (chartDecoding) { + m_chartManager = new ChartManager(*m_sentence, &system); + m_chartManager->ProcessSentence(); + ChartTrellisPathList nBestList; + m_chartManager->CalcNBest(nBestSize, nBestList, distinctNbest); + + cerr << "generate nbest list " << filename << endl; + cerr << "not implemented.." << endl; + exit(1); + if (filename != "") { + ofstream out(filename.c_str()); + if (!out) { + ostringstream msg; + msg << "Unable to open " << filename; + throw runtime_error(msg.str()); + } + // TODO: handle sentence id (for now always 0) +// OutputNBestList(const ChartTrellisPathList &nBestList, const ChartHypothesis *bestHypo, const TranslationSystem* system, long translationId, false) +// OutputNBest(out, nBestList, StaticData::Instance().GetOutputFactorOrder(),m_manager->GetTranslationSystem(), 0, false); + out.close(); + } + else { +// OutputNBest(streamOut, nBestList, StaticData::Instance().GetOutputFactorOrder(),m_manager->GetTranslationSystem(), sentenceid, false); + } + } + else { + // run the decoder + m_manager = new Moses::Manager(0,*m_sentence, staticData.GetSearchAlgorithm(), &system); + m_manager->ProcessSentence(); + TrellisPathList nBestList; + m_manager->CalcNBest(nBestSize, nBestList, distinctNbest); + + if (filename != "") { + ofstream out(filename.c_str()); + if (!out) { + ostringstream msg; + msg << "Unable to open " << filename; + throw runtime_error(msg.str()); + } + // TODO: handle sentence id (for now always 0) + //OutputNBest(out, nBestList, StaticData::Instance().GetOutputFactorOrder(),m_manager->GetTranslationSystem(), 0, false); + out.close(); + } + else { + //OutputNBest(streamOut, nBestList, StaticData::Instance().GetOutputFactorOrder(),m_manager->GetTranslationSystem(), sentenceid, false); + streamOut.flush(); + } + } + } + + void MosesDecoder::initialize(StaticData& staticData, const std::string& source, size_t sentenceid, + float bleuObjectiveWeight, float bleuScoreWeight, bool avgRefLength, bool chartDecoding) { + m_sentence = new Sentence(); + stringstream in(source + "\n"); + const std::vector<FactorType> &inputFactorOrder = staticData.GetInputFactorOrder(); + m_sentence->Read(in,inputFactorOrder); + + // set weight of BleuScoreFeature + //cerr << "Reload Bleu feature weight: " << bleuObjectiveWeight*bleuScoreWeight << " (" << bleuObjectiveWeight << "*" << bleuScoreWeight << ")" << endl; + staticData.ReLoadBleuScoreFeatureParameter(bleuObjectiveWeight*bleuScoreWeight); + + m_bleuScoreFeature->SetCurrSourceLength((*m_sentence).GetSize()); + if (chartDecoding) + m_bleuScoreFeature->SetCurrNormSourceLength((*m_sentence).GetSize()-2); + else + m_bleuScoreFeature->SetCurrNormSourceLength((*m_sentence).GetSize()); + + if (avgRefLength) + m_bleuScoreFeature->SetCurrAvgRefLength(sentenceid); + else + m_bleuScoreFeature->SetCurrShortestRefLength(sentenceid); + m_bleuScoreFeature->SetCurrReferenceNgrams(sentenceid); + } + + float MosesDecoder::getBleuScore(const ScoreComponentCollection& scores) { + return scores.GetScoreForProducer(m_bleuScoreFeature); + } + + void MosesDecoder::setBleuScore(ScoreComponentCollection& scores, float bleu) { + scores.Assign(m_bleuScoreFeature, bleu); + } + + ScoreComponentCollection MosesDecoder::getWeights() { + return StaticData::Instance().GetAllWeights(); + } + + void MosesDecoder::setWeights(const ScoreComponentCollection& weights) { + StaticData::InstanceNonConst().SetAllWeights(weights); + } + + void MosesDecoder::updateHistory(const vector<const Word*>& words) { + m_bleuScoreFeature->UpdateHistory(words); + } + + void MosesDecoder::updateHistory(const vector< vector< const Word*> >& words, vector<size_t>& sourceLengths, vector<size_t>& ref_ids, size_t rank, size_t epoch) { + m_bleuScoreFeature->UpdateHistory(words, sourceLengths, ref_ids, rank, epoch); + } + + void MosesDecoder::printBleuFeatureHistory(std::ostream& out) { + m_bleuScoreFeature->PrintHistory(out); + } + + size_t MosesDecoder::getClosestReferenceLength(size_t ref_id, int hypoLength) { + return m_bleuScoreFeature->GetClosestRefLength(ref_id, hypoLength); + } + + size_t MosesDecoder::getShortestReferenceIndex(size_t ref_id) { + return m_bleuScoreFeature->GetShortestRefIndex(ref_id); + } + + void MosesDecoder::setBleuParameters(bool disable, bool sentenceBleu, bool scaleByInputLength, bool scaleByAvgInputLength, + bool scaleByInverseLength, bool scaleByAvgInverseLength, + float scaleByX, float historySmoothing, size_t scheme, bool simpleHistoryBleu) { + m_bleuScoreFeature->SetBleuParameters(disable, sentenceBleu, scaleByInputLength, scaleByAvgInputLength, + scaleByInverseLength, scaleByAvgInverseLength, + scaleByX, historySmoothing, scheme, simpleHistoryBleu); + } +} + diff --git a/mira/Decoder.h b/mira/Decoder.h new file mode 100644 index 000000000..48b261fac --- /dev/null +++ b/mira/Decoder.h @@ -0,0 +1,140 @@ +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2010 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ +#ifndef _MIRA_DECODER_H_ +#define _MIRA_DECODER_H_ + +#include <iostream> +#include <cstring> +#include <sstream> + + +#include "BleuScoreFeature.h" +#include "ChartTrellisPathList.h" +#include "Hypothesis.h" +#include "Parameter.h" +#include "SearchNormal.h" +#include "Sentence.h" +#include "StaticData.h" + +// +// Wrapper functions and objects for the decoder. +// + +namespace Mira { + +/** + * Wraps moses decoder. + **/ +class MosesDecoder { + public: + /** + * Initialise moses (including StaticData) using the given ini file and debuglevel, passing through any + * other command line arguments. + **/ + MosesDecoder(const std::string& inifile, int debuglevel, int argc, std::vector<std::string> decoder_params); + + //returns the best sentence + std::vector< std::vector<const Moses::Word*> > getNBest(const std::string& source, + size_t sentenceid, + size_t nbestSize, + float bleuObjectiveweight, //weight of bleu in objective + float bleuScoreWeight, //weight of bleu in score + std::vector< Moses::ScoreComponentCollection>& featureValues, + std::vector< float>& bleuScores, + std::vector< float>& modelScores, + size_t numReturnedTranslations, + bool realBleu, + bool distinct, + bool avgRefLength, + size_t rank, + size_t epoch, + std::string filename); + std::vector< std::vector<const Moses::Word*> > runDecoder(const std::string& source, + size_t sentenceid, + size_t nbestSize, + float bleuObjectiveweight, //weight of bleu in objective + float bleuScoreWeight, //weight of bleu in score + std::vector< Moses::ScoreComponentCollection>& featureValues, + std::vector< float>& bleuScores, + std::vector< float>& modelScores, + size_t numReturnedTranslations, + bool realBleu, + bool distinct, + size_t rank, + size_t epoch, + Moses::SearchAlgorithm& seach, + const Moses::TranslationSystem& system, + std::string filename); + std::vector< std::vector<const Moses::Word*> > runChartDecoder(const std::string& source, + size_t sentenceid, + size_t nbestSize, + float bleuObjectiveweight, //weight of bleu in objective + float bleuScoreWeight, //weight of bleu in score + std::vector< Moses::ScoreComponentCollection>& featureValues, + std::vector< float>& bleuScores, + std::vector< float>& modelScores, + size_t numReturnedTranslations, + bool realBleu, + bool distinct, + size_t rank, + size_t epoch, + const Moses::TranslationSystem& system); + void outputNBestList(const std::string& source, + size_t sentenceid, + size_t nBestSize, + float bleuObjectiveWeight, + float bleuScoreWeight, + bool distinctNbest, + bool avgRefLength, + std::string filename, + std::ofstream& streamOut); + void initialize(Moses::StaticData& staticData, const std::string& source, size_t sentenceid, + float bleuObjectiveWeight, float bleuScoreWeight, bool avgRefLength, bool chartDecoding); + void updateHistory(const std::vector<const Moses::Word*>& words); + void updateHistory(const std::vector< std::vector< const Moses::Word*> >& words, std::vector<size_t>& sourceLengths, std::vector<size_t>& ref_ids, size_t rank, size_t epoch); + void printBleuFeatureHistory(std::ostream& out); + void printReferenceLength(const std::vector<size_t>& ref_ids); + size_t getReferenceLength(size_t ref_id); + size_t getClosestReferenceLength(size_t ref_id, int hypoLength); + size_t getShortestReferenceIndex(size_t ref_id); + void setBleuParameters(bool disable, bool sentenceBleu, bool scaleByInputLength, bool scaleByAvgInputLength, + bool scaleByInverseLength, bool scaleByAvgInverseLength, + float scaleByX, float historySmoothing, size_t scheme, bool simpleHistoryBleu); + void setAvgInputLength (float l) { m_bleuScoreFeature->SetAvgInputLength(l); } + Moses::ScoreComponentCollection getWeights(); + void setWeights(const Moses::ScoreComponentCollection& weights); + void cleanup(bool chartDecoding); + + float getSourceLengthHistory() { return m_bleuScoreFeature->GetSourceLengthHistory(); } + float getTargetLengthHistory() { return m_bleuScoreFeature->GetTargetLengthHistory(); } + float getAverageInputLength() { return m_bleuScoreFeature->GetAverageInputLength(); } + + private: + float getBleuScore(const Moses::ScoreComponentCollection& scores); + void setBleuScore(Moses::ScoreComponentCollection& scores, float bleu); + Moses::Manager *m_manager; + Moses::ChartManager *m_chartManager; + Moses::Sentence *m_sentence; + Moses::BleuScoreFeature *m_bleuScoreFeature; +}; + + +} //namespace + +#endif diff --git a/mira/Hildreth.cpp b/mira/Hildreth.cpp new file mode 100644 index 000000000..53d1e0881 --- /dev/null +++ b/mira/Hildreth.cpp @@ -0,0 +1,188 @@ +#include "Hildreth.h" + +using namespace Moses; +using namespace std; + +namespace Mira { + + vector<float> Hildreth::optimise (const vector<ScoreComponentCollection>& a, const vector<float>& b) { + + size_t i; + int max_iter = 10000; + float eps = 0.00000001; + float zero = 0.000000000001; + + vector<float> alpha ( b.size() ); + vector<float> F ( b.size() ); + vector<float> kkt ( b.size() ); + + float max_kkt = -1e100; + + size_t K = b.size(); + + float A[K][K]; + bool is_computed[K]; + for ( i = 0; i < K; i++ ) + { + A[i][i] = a[i].InnerProduct(a[i]); + is_computed[i] = false; + } + + int max_kkt_i = -1; + + + for ( i = 0; i < b.size(); i++ ) + { + F[i] = b[i]; + kkt[i] = F[i]; + if ( kkt[i] > max_kkt ) + { + max_kkt = kkt[i]; + max_kkt_i = i; + } + } + + int iter = 0; + float diff_alpha; + float try_alpha; + float add_alpha; + + while ( max_kkt >= eps && iter < max_iter ) + { + + diff_alpha = A[max_kkt_i][max_kkt_i] <= zero ? 0.0 : F[max_kkt_i]/A[max_kkt_i][max_kkt_i]; + try_alpha = alpha[max_kkt_i] + diff_alpha; + add_alpha = 0.0; + + if ( try_alpha < 0.0 ) + add_alpha = -1.0 * alpha[max_kkt_i]; + else + add_alpha = diff_alpha; + + alpha[max_kkt_i] = alpha[max_kkt_i] + add_alpha; + + if ( !is_computed[max_kkt_i] ) + { + for ( i = 0; i < K; i++ ) + { + A[i][max_kkt_i] = a[i].InnerProduct(a[max_kkt_i] ); // for version 1 + //A[i][max_kkt_i] = 0; // for version 1 + is_computed[max_kkt_i] = true; + } + } + + for ( i = 0; i < F.size(); i++ ) + { + F[i] -= add_alpha * A[i][max_kkt_i]; + kkt[i] = F[i]; + if ( alpha[i] > zero ) + kkt[i] = abs ( F[i] ); + } + max_kkt = -1e100; + max_kkt_i = -1; + for ( i = 0; i < F.size(); i++ ) + if ( kkt[i] > max_kkt ) + { + max_kkt = kkt[i]; + max_kkt_i = i; + } + + iter++; + } + + return alpha; + } + + vector<float> Hildreth::optimise (const vector<ScoreComponentCollection>& a, const vector<float>& b, float C) { + + size_t i; + int max_iter = 10000; + float eps = 0.00000001; + float zero = 0.000000000001; + + vector<float> alpha ( b.size() ); + vector<float> F ( b.size() ); + vector<float> kkt ( b.size() ); + + float max_kkt = -1e100; + + size_t K = b.size(); + + float A[K][K]; + bool is_computed[K]; + for ( i = 0; i < K; i++ ) + { + A[i][i] = a[i].InnerProduct(a[i]); + is_computed[i] = false; + } + + int max_kkt_i = -1; + + + for ( i = 0; i < b.size(); i++ ) + { + F[i] = b[i]; + kkt[i] = F[i]; + if ( kkt[i] > max_kkt ) + { + max_kkt = kkt[i]; + max_kkt_i = i; + } + } + + int iter = 0; + float diff_alpha; + float try_alpha; + float add_alpha; + + while ( max_kkt >= eps && iter < max_iter ) + { + + diff_alpha = A[max_kkt_i][max_kkt_i] <= zero ? 0.0 : F[max_kkt_i]/A[max_kkt_i][max_kkt_i]; + try_alpha = alpha[max_kkt_i] + diff_alpha; + add_alpha = 0.0; + + if ( try_alpha < 0.0 ) + add_alpha = -1.0 * alpha[max_kkt_i]; + else if (try_alpha > C) + add_alpha = C - alpha[max_kkt_i]; + else + add_alpha = diff_alpha; + + alpha[max_kkt_i] = alpha[max_kkt_i] + add_alpha; + + if ( !is_computed[max_kkt_i] ) + { + for ( i = 0; i < K; i++ ) + { + A[i][max_kkt_i] = a[i].InnerProduct(a[max_kkt_i] ); // for version 1 + //A[i][max_kkt_i] = 0; // for version 1 + is_computed[max_kkt_i] = true; + } + } + + for ( i = 0; i < F.size(); i++ ) + { + F[i] -= add_alpha * A[i][max_kkt_i]; + kkt[i] = F[i]; + if (alpha[i] > C - zero) + kkt[i]=-kkt[i]; + else if (alpha[i] > zero) + kkt[i] = abs(F[i]); + + } + max_kkt = -1e100; + max_kkt_i = -1; + for ( i = 0; i < F.size(); i++ ) + if ( kkt[i] > max_kkt ) + { + max_kkt = kkt[i]; + max_kkt_i = i; + } + + iter++; + } + + return alpha; + } +} diff --git a/mira/Hildreth.h b/mira/Hildreth.h new file mode 100644 index 000000000..699115242 --- /dev/null +++ b/mira/Hildreth.h @@ -0,0 +1,11 @@ +#include "FeatureVector.h" +#include "ScoreComponentCollection.h" + +namespace Mira { + + class Hildreth { + public : + static std::vector<float> optimise (const std::vector<Moses::ScoreComponentCollection>& a, const std::vector<float>& b ); + static std::vector<float> optimise (const std::vector<Moses::ScoreComponentCollection>& a, const std::vector<float>& b, float C); + }; +} diff --git a/mira/HildrethTest.cpp b/mira/HildrethTest.cpp new file mode 100644 index 000000000..a32dcd1d3 --- /dev/null +++ b/mira/HildrethTest.cpp @@ -0,0 +1,784 @@ +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2010 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#include <cstdio> +#include <cstdlib> +#include <string> + +#include <boost/test/unit_test.hpp> + +#include "Hildreth.h" +#include "Optimiser.h" +#include "ScoreComponentCollection.h" + +using namespace std; +using namespace Moses; +using namespace Mira; + +namespace MosesTest +{ + +class MockSingleFeature : public StatelessFeatureFunction { + public: + MockSingleFeature(): StatelessFeatureFunction("MockSingle",1) {} + std::string GetScoreProducerWeightShortName(unsigned) const {return "sf";} +}; + +class MockMultiFeature : public StatelessFeatureFunction { + public: + MockMultiFeature(): StatelessFeatureFunction("MockMulti",5) {} + std::string GetScoreProducerWeightShortName(unsigned) const {return "mf";} +}; + +class MockSparseFeature : public StatelessFeatureFunction { + public: + MockSparseFeature(): StatelessFeatureFunction("MockSparse", ScoreProducer::unlimited) {} + std::string GetScoreProducerWeightShortName(unsigned) const {return "sf";} +}; + +struct MockProducers { + MockProducers() {} + + MockSingleFeature single; + MockMultiFeature multi; + MockSparseFeature sparse; +}; + + + +BOOST_AUTO_TEST_SUITE(hildreth_test) + +BOOST_FIXTURE_TEST_CASE(test_hildreth_1, MockProducers) +{ + // Feasible example with 2 constraints + cerr << "\n>>>>>Hildreth test, without slack and with 0.01 slack" << endl << endl; + vector< ScoreComponentCollection> featureValueDiffs; + vector< float> lossMinusModelScoreDiff; + + // initial weights + float w[] = { 1, 1, 1, 1, 0 }; + vector<float> vec(w,w+5); + ScoreComponentCollection weights; + weights.PlusEquals(&multi, vec); + + // feature values (second is oracle) + //float arr1[] = {0, -5, -27.0908, -1.83258, 0 }; + //float arr2[] = {0, -5, -29.158, -1.83258, 0 }; + //float arr3[] = {0, -5, -27.0908, -1.83258, 0 }; + + // feature value differences (to oracle) + ScoreComponentCollection s1, s2, s3; + float arr1[] = { 0, 0, -2.0672, 0, 0 }; + float arr2[] = { 0, 0, 0, 0, 0 }; + float arr3[] = { 0, 0, -2.0672, 0, 0 }; + + float loss1 = 2.34085; + float loss2 = 0; + float loss3 = 2.34085; + + vector<float> vec1(arr1,arr1+5); + vector<float> vec2(arr2,arr2+5); + vector<float> vec3(arr3,arr3+5); + + s1.PlusEquals(&multi,vec1); + s2.PlusEquals(&multi,vec2); + s3.PlusEquals(&multi,vec3); + + featureValueDiffs.push_back(s1); + featureValueDiffs.push_back(s2); + featureValueDiffs.push_back(s3); + + cerr << "feature value diff: " << featureValueDiffs[0] << endl; + cerr << "feature value diff: " << featureValueDiffs[1] << endl; + cerr << "feature value diff: " << featureValueDiffs[2] << endl << endl; + + float oldModelScoreDiff1 = featureValueDiffs[0].InnerProduct(weights); + float oldModelScoreDiff2 = featureValueDiffs[1].InnerProduct(weights); + float oldModelScoreDiff3 = featureValueDiffs[2].InnerProduct(weights); + + cerr << "model score diff: " << oldModelScoreDiff1 << ", loss: " << loss1 << endl; + cerr << "model score diff: " << oldModelScoreDiff2 << ", loss: " << loss2 << endl; + cerr << "model score diff: " << oldModelScoreDiff3 << ", loss: " << loss3 << endl << endl; + + lossMinusModelScoreDiff.push_back(loss1 - oldModelScoreDiff1); + lossMinusModelScoreDiff.push_back(loss2 - oldModelScoreDiff2); + lossMinusModelScoreDiff.push_back(loss3 - oldModelScoreDiff3); + + vector< float> alphas1 = Hildreth::optimise(featureValueDiffs, lossMinusModelScoreDiff); + vector< float> alphas2 = Hildreth::optimise(featureValueDiffs, lossMinusModelScoreDiff, 0.01); + + cerr << "\nalphas without slack:" << endl; + for (size_t i = 0; i < alphas1.size(); ++i) { + cerr << "alpha " << i << ": " << alphas1[i] << endl; + } + cerr << endl; + + cerr << "partial updates:" << endl; + vector< ScoreComponentCollection> featureValueDiffs1(featureValueDiffs); + FVector totalUpdate1 = ScoreComponentCollection::CreateFVector(); + for (size_t k = 0; k < featureValueDiffs1.size(); ++k) { + featureValueDiffs1[k].MultiplyEquals(alphas1[k]); + cerr << k << ": " << featureValueDiffs1[k].GetScoresVector() << endl; + FVector update = featureValueDiffs1[k].GetScoresVector(); + totalUpdate1 += update; + } + cerr << endl; + cerr << "total update: " << totalUpdate1 << endl << endl; + + ScoreComponentCollection weightsUpdate1(weights); + weightsUpdate1.PlusEquals(totalUpdate1); + cerr << "new weights: " << weightsUpdate1 << endl << endl; + + float newModelScoreDiff1 = featureValueDiffs[0].InnerProduct(weightsUpdate1); + float newModelScoreDiff2 = featureValueDiffs[1].InnerProduct(weightsUpdate1); + float newModelScoreDiff3 = featureValueDiffs[2].InnerProduct(weightsUpdate1); + + cerr << "new model score diff: " << newModelScoreDiff1 << ", loss: " << loss1 << endl; + cerr << "new model score diff: " << newModelScoreDiff2 << ", loss: " << loss2 << endl; + cerr << "new model score diff: " << newModelScoreDiff3 << ", loss: " << loss3 << endl; + + cerr << "\n\nalphas with slack 0.01:" << endl; + for (size_t i = 0; i < alphas2.size(); ++i) { + cerr << "alpha " << i << ": " << alphas2[i] << endl; + } + cerr << endl; + + cerr << "partial updates:" << endl; + vector< ScoreComponentCollection> featureValueDiffs2(featureValueDiffs); + FVector totalUpdate2 = ScoreComponentCollection::CreateFVector(); + for (size_t k = 0; k < featureValueDiffs2.size(); ++k) { + featureValueDiffs2[k].MultiplyEquals(alphas2[k]); + cerr << k << ": " << featureValueDiffs2[k].GetScoresVector() << endl; + FVector update = featureValueDiffs2[k].GetScoresVector(); + totalUpdate2 += update; + } + cerr << endl; + cerr << "total update: " << totalUpdate2 << endl << endl; + + ScoreComponentCollection weightsUpdate2(weights); + weightsUpdate2.PlusEquals(totalUpdate2); + cerr << "new weights: " << weightsUpdate2 << endl << endl; + + float newModelScoreDiff4 = featureValueDiffs[0].InnerProduct(weightsUpdate2); + float newModelScoreDiff5 = featureValueDiffs[1].InnerProduct(weightsUpdate2); + float newModelScoreDiff6 = featureValueDiffs[2].InnerProduct(weightsUpdate2); + + cerr << "new model score diff: " << newModelScoreDiff4 << ", loss: " << loss1 << endl; + cerr << "new model score diff: " << newModelScoreDiff5 << ", loss: " << loss2 << endl; + cerr << "new model score diff: " << newModelScoreDiff6 << ", loss: " << loss3 << endl; +} + + +BOOST_FIXTURE_TEST_CASE(test_hildreth_3, MockProducers) +{ + // Unfeasible example with 21 constraints + cerr << "\n>>>>>Hildreth test, without slack and with 0.01 slack" << endl << endl; + vector< ScoreComponentCollection> featureValueDiffs; + vector< float> lossMinusModelScoreDiff; + + // initial weights + float w[] = { 1, 1, 0.638672, 1, 0 }; + vector<float> vec(w,w+5); + ScoreComponentCollection weights; + weights.PlusEquals(&multi, vec); + + int numberOfConstraints = 21; + + // feature value differences (to oracle) + // NOTE: these feature values are only approximations + ScoreComponentCollection s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14, s15, s16, s17, s18, s19, s20, s21; + float arr1[] = { 0, 0, -2.0672, 0, 0 }; + float arr2[] = { 0, 0, 0, 0, 0 }; + float arr3[] = { 0, 0, -2.08436, 1.38629, 0 }; + float arr4[] = { 0, 0, -0.0171661, 1.38629, 0 }; + float arr5[] = { 0, 0, 4.4283, 0, 0 }; + float arr6[] = { 0, 0, 3.84829, 1.38629, 0 }; + float arr7[] = { 0, 0, 6.83689, 0, 0 }; + float arr8[] = { 0, 0, 0, 0, 0 }; + float arr9[] = { 0, 0, -2.0672, 0, 0 }; + float arr10[] = { 0, 0, -0.0171661, 1.38629, 0 }; + float arr11[] = { 0, 0, -2.08436, 1.38629, 0 }; + float arr12[] = { 0, 0, 4.4283, 0, 0 }; + float arr13[] = { 3, 0, 2.41089, 0, 0 }; + float arr14[] = { 3, 0, 2.32709, 0, 0 }; + float arr15[] = { 0, 0, -2.0672, 0, 0 }; + float arr16[] = { 0, 0, -2.08436, 1.38629, 0 }; + float arr17[] = { 0, 0, 4.4283, 0, 0 }; + float arr18[] = { 0, 0, 3.84829, 1.38629, 0 }; + float arr19[] = { 0, 0, -0.0171661, 1.38629, 0 }; + float arr20[] = { 0, 0, 0, 0, 0 }; + float arr21[] = { 0, 0, 6.83689, 0, 0 }; + + vector<float> losses; + losses.push_back(2.73485); + losses.push_back(0); + losses.push_back(3.64118); + losses.push_back(1.47347); + losses.push_back(3.64118); + losses.push_back(4.16278); + losses.push_back(3.13952); + losses.push_back(0); + losses.push_back(2.73485); + losses.push_back(1.47347); + losses.push_back(3.64118); + losses.push_back(3.64118); + losses.push_back(2.51662); + losses.push_back(2.73485); + losses.push_back(2.73485); + losses.push_back(3.64118); + losses.push_back(3.64118); + losses.push_back(4.16278); + losses.push_back(1.47347); + losses.push_back(0); + losses.push_back(3.13952); + + vector<float> vec1(arr1,arr1+5); + vector<float> vec2(arr2,arr2+5); + vector<float> vec3(arr3,arr3+5); + vector<float> vec4(arr4,arr4+5); + vector<float> vec5(arr5,arr5+5); + vector<float> vec6(arr6,arr6+5); + vector<float> vec7(arr7,arr7+5); + vector<float> vec8(arr8,arr8+5); + vector<float> vec9(arr9,arr9+5); + vector<float> vec10(arr10,arr10+5); + vector<float> vec11(arr11,arr11+5); + vector<float> vec12(arr12,arr12+5); + vector<float> vec13(arr13,arr13+5); + vector<float> vec14(arr14,arr14+5); + vector<float> vec15(arr15,arr15+5); + vector<float> vec16(arr16,arr16+5); + vector<float> vec17(arr17,arr17+5); + vector<float> vec18(arr18,arr18+5); + vector<float> vec19(arr19,arr19+5); + vector<float> vec20(arr20,arr20+5); + vector<float> vec21(arr21,arr21+5); + + s1.PlusEquals(&multi,vec1); + s2.PlusEquals(&multi,vec2); + s3.PlusEquals(&multi,vec3); + s4.PlusEquals(&multi,vec4); + s5.PlusEquals(&multi,vec5); + s6.PlusEquals(&multi,vec6); + s7.PlusEquals(&multi,vec7); + s8.PlusEquals(&multi,vec8); + s9.PlusEquals(&multi,vec9); + s10.PlusEquals(&multi,vec10); + s11.PlusEquals(&multi,vec11); + s12.PlusEquals(&multi,vec12); + s13.PlusEquals(&multi,vec13); + s14.PlusEquals(&multi,vec14); + s15.PlusEquals(&multi,vec15); + s16.PlusEquals(&multi,vec16); + s17.PlusEquals(&multi,vec17); + s18.PlusEquals(&multi,vec18); + s19.PlusEquals(&multi,vec19); + s20.PlusEquals(&multi,vec20); + s21.PlusEquals(&multi,vec21); + + featureValueDiffs.push_back(s1); + featureValueDiffs.push_back(s2); + featureValueDiffs.push_back(s3); + featureValueDiffs.push_back(s4); + featureValueDiffs.push_back(s5); + featureValueDiffs.push_back(s6); + featureValueDiffs.push_back(s7); + featureValueDiffs.push_back(s8); + featureValueDiffs.push_back(s9); + featureValueDiffs.push_back(s10); + featureValueDiffs.push_back(s11); + featureValueDiffs.push_back(s12); + featureValueDiffs.push_back(s13); + featureValueDiffs.push_back(s14); + featureValueDiffs.push_back(s15); + featureValueDiffs.push_back(s16); + featureValueDiffs.push_back(s17); + featureValueDiffs.push_back(s18); + featureValueDiffs.push_back(s19); + featureValueDiffs.push_back(s20); + featureValueDiffs.push_back(s21); + + vector<float> oldModelScoreDiff; + for (int i = 0; i < numberOfConstraints; ++i) { + oldModelScoreDiff.push_back(featureValueDiffs[i].InnerProduct(weights)); + } + + for (int i = 0; i < numberOfConstraints; ++i) { + cerr << "old model score diff: " << oldModelScoreDiff[i] << ", loss: " << losses[i] << "\t" << (oldModelScoreDiff[i] >= losses[i] ? 1 : 0) << endl; + } + + for (int i = 0; i < numberOfConstraints; ++i) { + lossMinusModelScoreDiff.push_back(losses[i] - oldModelScoreDiff[i]); + } + + for (int i = 0; i < numberOfConstraints; ++i) { + cerr << "A: " << featureValueDiffs[i] << ", b: " << lossMinusModelScoreDiff[i] << endl; + } + + vector< float> alphas1 = Hildreth::optimise(featureValueDiffs, lossMinusModelScoreDiff); + vector< float> alphas2 = Hildreth::optimise(featureValueDiffs, lossMinusModelScoreDiff, 0.01); + + cerr << "\nalphas without slack:" << endl; + for (size_t i = 0; i < alphas1.size(); ++i) { + cerr << "alpha " << i << ": " << alphas1[i] << endl; + } + cerr << endl; + + cerr << "partial updates:" << endl; + vector< ScoreComponentCollection> featureValueDiffs1(featureValueDiffs); + FVector totalUpdate1 = ScoreComponentCollection::CreateFVector(); + for (size_t k = 0; k < featureValueDiffs1.size(); ++k) { + featureValueDiffs1[k].MultiplyEquals(alphas1[k]); + cerr << k << ": " << featureValueDiffs1[k].GetScoresVector() << endl; + FVector update = featureValueDiffs1[k].GetScoresVector(); + totalUpdate1 += update; + } + cerr << endl; + cerr << "total update: " << totalUpdate1 << endl << endl; + + ScoreComponentCollection weightsUpdate1(weights); + weightsUpdate1.PlusEquals(totalUpdate1); + cerr << "old weights: " << weights << endl; + cerr << "new weights: " << weightsUpdate1 << endl << endl; + + vector<float> newModelScoreDiff; + for (int i = 0; i < numberOfConstraints; ++i) { + newModelScoreDiff.push_back(featureValueDiffs[i].InnerProduct(weightsUpdate1)); + } + + for (int i = 0; i < numberOfConstraints; ++i) { + cerr << "new model score diff: " << newModelScoreDiff[i] << ", loss: " << losses[i] << "\t" << (newModelScoreDiff[i] >= losses[i] ? 1 : 0) << endl; + } + + cerr << "\n\nalphas with slack 0.01:" << endl; + for (size_t i = 0; i < alphas2.size(); ++i) { + cerr << "alpha " << i << ": " << alphas2[i] << endl; + } + cerr << endl; + + cerr << "partial updates:" << endl; + vector< ScoreComponentCollection> featureValueDiffs2(featureValueDiffs); + FVector totalUpdate2 = ScoreComponentCollection::CreateFVector(); + for (size_t k = 0; k < featureValueDiffs2.size(); ++k) { + featureValueDiffs2[k].MultiplyEquals(alphas2[k]); + cerr << k << ": " << featureValueDiffs2[k].GetScoresVector() << endl; + FVector update = featureValueDiffs2[k].GetScoresVector(); + totalUpdate2 += update; + } + cerr << endl; + cerr << "total update: " << totalUpdate2 << endl << endl; + + ScoreComponentCollection weightsUpdate2(weights); + weightsUpdate2.PlusEquals(totalUpdate2); + cerr << "old weights: " << weights << endl; + cerr << "new weights: " << weightsUpdate2 << endl << endl; + + newModelScoreDiff.clear(); + for (int i = 0; i < numberOfConstraints; ++i) { + newModelScoreDiff.push_back(featureValueDiffs[i].InnerProduct(weightsUpdate2)); + } + + for (int i = 0; i < numberOfConstraints; ++i) { + cerr << "new model score diff: " << newModelScoreDiff[i] << ", loss: " << losses[i] << endl; + } +} + +BOOST_FIXTURE_TEST_CASE(test_hildreth_4, MockProducers) +{ + // Feasible example with 8 constraints + cerr << "\n>>>>>Hildreth test, without slack and with 0.01 slack" << endl << endl; + vector< ScoreComponentCollection> featureValueDiffs; + vector< float> lossMinusModelScoreDiff; + + // initial weights + float w[] = { 1, 1, 0.638672, 1, 0 }; + vector<float> vec(w,w+5); + ScoreComponentCollection weights; + weights.PlusEquals(&multi, vec); + + int numberOfConstraints = 8; + + // feature value differences (to oracle) + // NOTE: these feature values are only approximations + ScoreComponentCollection s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, s12, s13, s14, s15, s16, s17, s18, s19, s20, s21; + float arr1[] = { 0, 0, -2.0672, 0, 0 }; + float arr2[] = { 0, 0, 0, 0, 0 }; + float arr3[] = { 0, 0, -2.08436, 1.38629, 0 }; + float arr4[] = { 0, 0, -0.0171661, 1.38629, 0 }; +// float arr5[] = { 0, 0, 4.4283, 0, 0 }; +// float arr6[] = { 0, 0, 3.84829, 1.38629, 0 }; +// float arr7[] = { 0, 0, 6.83689, 0, 0 }; + + float arr8[] = { 0, 0, 0, 0, 0 }; + float arr9[] = { 0, 0, -2.0672, 0, 0 }; +// float arr10[] = { 0, 0, -0.0171661, 1.38629, 0 }; +// float arr11[] = { 0, 0, -2.08436, 1.38629, 0 }; +// float arr12[] = { 0, 0, 4.4283, 0, 0 }; +// float arr13[] = { 3, 0, 2.41089, 0, 0 }; +// float arr14[] = { 3, 0, 2.32709, 0, 0 }; + + float arr15[] = { 0, 0, -2.0672, 0, 0 }; + float arr16[] = { 0, 0, -2.08436, 1.38629, 0 }; +// float arr17[] = { 0, 0, 4.4283, 0, 0 }; +// float arr18[] = { 0, 0, 3.84829, 1.38629, 0 }; +// float arr19[] = { 0, 0, -0.0171661, 1.38629, 0 }; +// float arr20[] = { 0, 0, 0, 0, 0 }; +// float arr21[] = { 0, 0, 6.83689, 0, 0 }; + + vector<float> losses; + losses.push_back(2.73485); + losses.push_back(0); + losses.push_back(3.64118); + losses.push_back(1.47347); +// losses.push_back(3.64118); +// losses.push_back(4.16278); +// losses.push_back(3.13952); + losses.push_back(0); + losses.push_back(2.73485); +// losses.push_back(1.47347); +// losses.push_back(3.64118); +// losses.push_back(3.64118); +// losses.push_back(2.51662); +// losses.push_back(2.73485); + losses.push_back(2.73485); + losses.push_back(3.64118); +// losses.push_back(3.64118); +// losses.push_back(4.16278); +// losses.push_back(1.47347); +// losses.push_back(0); +// losses.push_back(3.13952); + + vector<float> vec1(arr1,arr1+5); + vector<float> vec2(arr2,arr2+5); + vector<float> vec3(arr3,arr3+5); + vector<float> vec4(arr4,arr4+5); +// vector<float> vec5(arr5,arr5+5); +// vector<float> vec6(arr6,arr6+5); +// vector<float> vec7(arr7,arr7+5); + vector<float> vec8(arr8,arr8+5); + vector<float> vec9(arr9,arr9+5); +// vector<float> vec10(arr10,arr10+5); +// vector<float> vec11(arr11,arr11+5); +// vector<float> vec12(arr12,arr12+5); +// vector<float> vec13(arr13,arr13+5); +// vector<float> vec14(arr14,arr14+5); + vector<float> vec15(arr15,arr15+5); + vector<float> vec16(arr16,arr16+5); +// vector<float> vec17(arr17,arr17+5); +// vector<float> vec18(arr18,arr18+5); +// vector<float> vec19(arr19,arr19+5); +// vector<float> vec20(arr20,arr20+5); +// vector<float> vec21(arr21,arr21+5); + + s1.PlusEquals(&multi,vec1); + s2.PlusEquals(&multi,vec2); + s3.PlusEquals(&multi,vec3); + s4.PlusEquals(&multi,vec4); +// s5.PlusEquals(&multi,vec5); +// s6.PlusEquals(&multi,vec6); +// s7.PlusEquals(&multi,vec7); + s8.PlusEquals(&multi,vec8); + s9.PlusEquals(&multi,vec9); +// s10.PlusEquals(&multi,vec10); +// s11.PlusEquals(&multi,vec11); +// s12.PlusEquals(&multi,vec12); +// s13.PlusEquals(&multi,vec13); +// s14.PlusEquals(&multi,vec14); + s15.PlusEquals(&multi,vec15); + s16.PlusEquals(&multi,vec16); +// s17.PlusEquals(&multi,vec17); +// s18.PlusEquals(&multi,vec18); +// s19.PlusEquals(&multi,vec19); +// s20.PlusEquals(&multi,vec20); +// s21.PlusEquals(&multi,vec21); + + featureValueDiffs.push_back(s1); + featureValueDiffs.push_back(s2); + featureValueDiffs.push_back(s3); + featureValueDiffs.push_back(s4); +// featureValueDiffs.push_back(s5); +// featureValueDiffs.push_back(s6); +// featureValueDiffs.push_back(s7); + featureValueDiffs.push_back(s8); + featureValueDiffs.push_back(s9); +// featureValueDiffs.push_back(s10); +// featureValueDiffs.push_back(s11); +// featureValueDiffs.push_back(s12); +// featureValueDiffs.push_back(s13); +// featureValueDiffs.push_back(s14); + featureValueDiffs.push_back(s15); + featureValueDiffs.push_back(s16); +// featureValueDiffs.push_back(s17); +// featureValueDiffs.push_back(s18); +// featureValueDiffs.push_back(s19); +// featureValueDiffs.push_back(s20); +// featureValueDiffs.push_back(s21); + + vector<float> oldModelScoreDiff; + for (int i = 0; i < numberOfConstraints; ++i) { + oldModelScoreDiff.push_back(featureValueDiffs[i].InnerProduct(weights)); + } + + for (int i = 0; i < numberOfConstraints; ++i) { + cerr << "old model score diff: " << oldModelScoreDiff[i] << ", loss: " << losses[i] << "\t" << (oldModelScoreDiff[i] >= losses[i] ? 1 : 0) << endl; + } + + for (int i = 0; i < numberOfConstraints; ++i) { + lossMinusModelScoreDiff.push_back(losses[i] - oldModelScoreDiff[i]); + } + + for (int i = 0; i < numberOfConstraints; ++i) { + cerr << "A: " << featureValueDiffs[i] << ", b: " << lossMinusModelScoreDiff[i] << endl; + } + + vector< float> alphas1 = Hildreth::optimise(featureValueDiffs, lossMinusModelScoreDiff); + vector< float> alphas2 = Hildreth::optimise(featureValueDiffs, lossMinusModelScoreDiff, 0.01); + + cerr << "\nalphas without slack:" << endl; + for (size_t i = 0; i < alphas1.size(); ++i) { + cerr << "alpha " << i << ": " << alphas1[i] << endl; + } + cerr << endl; + + cerr << "partial updates:" << endl; + vector< ScoreComponentCollection> featureValueDiffs1(featureValueDiffs); + FVector totalUpdate1 = ScoreComponentCollection::CreateFVector(); + for (size_t k = 0; k < featureValueDiffs1.size(); ++k) { + featureValueDiffs1[k].MultiplyEquals(alphas1[k]); + cerr << k << ": " << featureValueDiffs1[k].GetScoresVector() << endl; + FVector update = featureValueDiffs1[k].GetScoresVector(); + totalUpdate1 += update; + } + cerr << endl; + cerr << "total update: " << totalUpdate1 << endl << endl; + + ScoreComponentCollection weightsUpdate1(weights); + weightsUpdate1.PlusEquals(totalUpdate1); + cerr << "old weights: " << weights << endl; + cerr << "new weights: " << weightsUpdate1 << endl << endl; + + vector<float> newModelScoreDiff; + for (int i = 0; i < numberOfConstraints; ++i) { + newModelScoreDiff.push_back(featureValueDiffs[i].InnerProduct(weightsUpdate1)); + } + + for (int i = 0; i < numberOfConstraints; ++i) { + cerr << "new model score diff: " << newModelScoreDiff[i] << ", loss: " << losses[i] << "\t" << (newModelScoreDiff[i] >= losses[i] ? 1 : 0) << endl; + } + + cerr << "\n\nalphas with slack 0.01:" << endl; + for (size_t i = 0; i < alphas2.size(); ++i) { + cerr << "alpha " << i << ": " << alphas2[i] << endl; + } + cerr << endl; + + cerr << "partial updates:" << endl; + vector< ScoreComponentCollection> featureValueDiffs2(featureValueDiffs); + FVector totalUpdate2 = ScoreComponentCollection::CreateFVector(); + for (size_t k = 0; k < featureValueDiffs2.size(); ++k) { + featureValueDiffs2[k].MultiplyEquals(alphas2[k]); + cerr << k << ": " << featureValueDiffs2[k].GetScoresVector() << endl; + FVector update = featureValueDiffs2[k].GetScoresVector(); + totalUpdate2 += update; + } + cerr << endl; + cerr << "total update: " << totalUpdate2 << endl << endl; + + ScoreComponentCollection weightsUpdate2(weights); + weightsUpdate2.PlusEquals(totalUpdate2); + cerr << "old weights: " << weights << endl; + cerr << "new weights: " << weightsUpdate2 << endl << endl; + + newModelScoreDiff.clear(); + for (int i = 0; i < numberOfConstraints; ++i) { + newModelScoreDiff.push_back(featureValueDiffs[i].InnerProduct(weightsUpdate2)); + } + + for (int i = 0; i < numberOfConstraints; ++i) { + cerr << "new model score diff: " << newModelScoreDiff[i] << ", loss: " << losses[i] << endl; + } +} + +BOOST_FIXTURE_TEST_CASE(test_hildreth_5, MockProducers) +{ + // Unfeasible example with 2 constraints + cerr << "\n>>>>>Hildreth test, without slack and with 0.01 slack" << endl << endl; + vector< ScoreComponentCollection> featureValueDiffs; + vector< float> lossMinusModelScoreDiff; + + // initial weights + float w[] = { 1, 1, 0.638672, 1, 0 }; + vector<float> vec(w,w+5); + ScoreComponentCollection weights; + weights.PlusEquals(&multi, vec); + + int numberOfConstraints = 2; + + // feature value differences (to oracle) + // NOTE: these feature values are only approximations + ScoreComponentCollection s1, s17; + float arr1[] = { 0, 0, -2.0672, 0, 0 }; + float arr17[] = { 0, 0, 4.4283, 0, 0 }; + vector<float> losses; + losses.push_back(2.73485); + losses.push_back(3.64118); + + vector<float> vec1(arr1,arr1+5); + vector<float> vec17(arr17,arr17+5); + + s1.PlusEquals(&multi,vec1); + s17.PlusEquals(&multi,vec17); + + featureValueDiffs.push_back(s1); + featureValueDiffs.push_back(s17); + + vector<float> oldModelScoreDiff; + for (int i = 0; i < numberOfConstraints; ++i) { + oldModelScoreDiff.push_back(featureValueDiffs[i].InnerProduct(weights)); + } + + float sumOfOldError = 0; + for (int i = 0; i < numberOfConstraints; ++i) { + cerr << "old model score diff: " << oldModelScoreDiff[i] << ", loss: " << losses[i] << "\t" << (oldModelScoreDiff[i] >= losses[i] ? 1 : 0) << endl; + sumOfOldError += (losses[i] - oldModelScoreDiff[i]); + } + cerr << "sum of old error: " << sumOfOldError << endl; + + for (int i = 0; i < numberOfConstraints; ++i) { + lossMinusModelScoreDiff.push_back(losses[i] - oldModelScoreDiff[i]); + } + + for (int i = 0; i < numberOfConstraints; ++i) { + cerr << "A: " << featureValueDiffs[i] << ", b: " << lossMinusModelScoreDiff[i] << endl; + } + + vector< float> alphas1 = Hildreth::optimise(featureValueDiffs, lossMinusModelScoreDiff); + vector< float> alphas2 = Hildreth::optimise(featureValueDiffs, lossMinusModelScoreDiff, 0.01); + vector< float> alphas3 = Hildreth::optimise(featureValueDiffs, lossMinusModelScoreDiff, 0.1); + + cerr << "\nalphas without slack:" << endl; + for (size_t i = 0; i < alphas1.size(); ++i) { + cerr << "alpha " << i << ": " << alphas1[i] << endl; + } + cerr << endl; + + cerr << "partial updates:" << endl; + vector< ScoreComponentCollection> featureValueDiffs1(featureValueDiffs); + FVector totalUpdate1 = ScoreComponentCollection::CreateFVector(); + for (size_t k = 0; k < featureValueDiffs1.size(); ++k) { + featureValueDiffs1[k].MultiplyEquals(alphas1[k]); + cerr << k << ": " << featureValueDiffs1[k].GetScoresVector() << endl; + FVector update = featureValueDiffs1[k].GetScoresVector(); + totalUpdate1 += update; + } + cerr << endl; + cerr << "total update: " << totalUpdate1 << endl << endl; + + ScoreComponentCollection weightsUpdate1(weights); + weightsUpdate1.PlusEquals(totalUpdate1); + cerr << "old weights: " << weights << endl; + cerr << "new weights: " << weightsUpdate1 << endl << endl; + + vector<float> newModelScoreDiff; + for (int i = 0; i < numberOfConstraints; ++i) { + newModelScoreDiff.push_back(featureValueDiffs[i].InnerProduct(weightsUpdate1)); + } + + float sumOfNewError = 0; + for (int i = 0; i < numberOfConstraints; ++i) { + cerr << "new model score diff: " << newModelScoreDiff[i] << ", loss: " << losses[i] << "\t" << (newModelScoreDiff[i] >= losses[i] ? 1 : 0) << endl; + sumOfNewError += (losses[i] - newModelScoreDiff[i]); + } + cerr << "sum of new error: " << sumOfNewError << endl; + + cerr << "\n\nalphas with slack 0.01:" << endl; + for (size_t i = 0; i < alphas2.size(); ++i) { + cerr << "alpha " << i << ": " << alphas2[i] << endl; + } + cerr << endl; + + cerr << "partial updates:" << endl; + vector< ScoreComponentCollection> featureValueDiffs2(featureValueDiffs); + FVector totalUpdate2 = ScoreComponentCollection::CreateFVector(); + for (size_t k = 0; k < featureValueDiffs2.size(); ++k) { + featureValueDiffs2[k].MultiplyEquals(alphas2[k]); + cerr << k << ": " << featureValueDiffs2[k].GetScoresVector() << endl; + FVector update = featureValueDiffs2[k].GetScoresVector(); + totalUpdate2 += update; + } + cerr << endl; + cerr << "total update: " << totalUpdate2 << endl << endl; + + ScoreComponentCollection weightsUpdate2(weights); + weightsUpdate2.PlusEquals(totalUpdate2); + cerr << "old weights: " << weights << endl; + cerr << "new weights: " << weightsUpdate2 << endl << endl; + + newModelScoreDiff.clear(); + for (int i = 0; i < numberOfConstraints; ++i) { + newModelScoreDiff.push_back(featureValueDiffs[i].InnerProduct(weightsUpdate2)); + } + + sumOfNewError = 0; + for (int i = 0; i < numberOfConstraints; ++i) { + cerr << "new model score diff: " << newModelScoreDiff[i] << ", loss: " << losses[i] << "\t" << (newModelScoreDiff[i] >= losses[i] ? 1 : 0) << endl; + sumOfNewError += (losses[i] - newModelScoreDiff[i]); + } + cerr << "sum of new error: " << sumOfNewError << endl; + + cerr << "\n\nalphas with slack 0.1:" << endl; + for (size_t i = 0; i < alphas3.size(); ++i) { + cerr << "alpha " << i << ": " << alphas3[i] << endl; + } + cerr << endl; + + cerr << "partial updates:" << endl; + vector< ScoreComponentCollection> featureValueDiffs3(featureValueDiffs); + FVector totalUpdate3 = ScoreComponentCollection::CreateFVector(); + for (size_t k = 0; k < featureValueDiffs3.size(); ++k) { + featureValueDiffs3[k].MultiplyEquals(alphas3[k]); + cerr << k << ": " << featureValueDiffs3[k].GetScoresVector() << endl; + FVector update = featureValueDiffs3[k].GetScoresVector(); + totalUpdate3 += update; + } + cerr << endl; + cerr << "total update: " << totalUpdate3 << endl << endl; + + ScoreComponentCollection weightsUpdate3(weights); + weightsUpdate3.PlusEquals(totalUpdate3); + cerr << "old weights: " << weights << endl; + cerr << "new weights: " << weightsUpdate3 << endl << endl; + + newModelScoreDiff.clear(); + for (int i = 0; i < numberOfConstraints; ++i) { + newModelScoreDiff.push_back(featureValueDiffs[i].InnerProduct(weightsUpdate3)); + } + + sumOfNewError = 0; + for (int i = 0; i < numberOfConstraints; ++i) { + cerr << "new model score diff: " << newModelScoreDiff[i] << ", loss: " << losses[i] << "\t" << (newModelScoreDiff[i] >= losses[i] ? 1 : 0) << endl; + sumOfNewError += (losses[i] - newModelScoreDiff[i]); + } + cerr << "sum of new error: " << sumOfNewError << endl; +} + +BOOST_AUTO_TEST_SUITE_END() + +} + diff --git a/mira/HypothesisQueue.cpp b/mira/HypothesisQueue.cpp new file mode 100644 index 000000000..43e082b92 --- /dev/null +++ b/mira/HypothesisQueue.cpp @@ -0,0 +1,62 @@ +/*********************************************************************** + Moses - statistical machine translation system + Copyright (C) 2006-2011 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#include <iostream> +#include "HypothesisQueue.h" + +using namespace std; + +namespace Moses { + +HypothesisQueue::~HypothesisQueue() { + m_queue.clear(); +} + +void HypothesisQueue::Push(BleuIndexPair hypo) { + //pair<set<BleuIndexPair>::iterator,bool> ret; + + if (m_capacity == 0 || m_queue.size() < m_capacity) { + m_queue.insert(hypo); + } else if (hypo.first > (*(m_queue.rbegin())).first) { + // Remove the worst-scoring item from the queue and insert hypo (only erase item if new item was successfully added ) + /*ret = m_queue.insert(hypo); + if ((*(ret.first)).second == 1) { + HypoQueueType::iterator p = m_queue.end(); + --p; + m_queue.erase(p); + }*/ + // with multisets we do not have to check whether the item was successfully added + m_queue.insert(hypo); + HypoQueueType::iterator p = m_queue.end(); + --p; + m_queue.erase(p); + } else { + // The hypo is unusable: the queue is full and hypo has a worse (or + // equal) score than the worst-scoring item already held. + } +} + +BleuIndexPair HypothesisQueue::Pop() { + HypoQueueType::iterator p = m_queue.begin(); + BleuIndexPair top = *p; + m_queue.erase(p); + return top; +} + +} // namespace Moses diff --git a/mira/HypothesisQueue.h b/mira/HypothesisQueue.h new file mode 100644 index 000000000..a926a40da --- /dev/null +++ b/mira/HypothesisQueue.h @@ -0,0 +1,65 @@ +/*********************************************************************** + Moses - statistical machine translation system + Copyright (C) 2006-2011 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#pragma once + +#include <set> + +namespace Moses { + +// pair of Bleu score and index +typedef std::pair<float, size_t> BleuIndexPair; + +// A bounded priority queue of BleuIndexPairs. The top item is +// the best scoring hypothesis. The queue assumes ownership of pushed items and +// relinquishes ownership when they are popped. Any remaining items at the +// time of the queue's destruction are deleted. +class HypothesisQueue { + + public: + // Create empty queue with fixed capacity of c. Capacity 0 means unbounded. + HypothesisQueue(size_t c) : m_capacity(c) {} + ~HypothesisQueue(); + + bool Empty() { return m_queue.empty(); } + + // Add the hypo to the queue or delete it if the queue is full and the + // score is no better than the queue's worst score. + void Push(BleuIndexPair hypo); + + // Remove the best-scoring detour from the queue and return it. The + // caller is responsible for deleting the object. + BleuIndexPair Pop(); + + private: + struct HypothesisOrderer { + bool operator()(BleuIndexPair a, + BleuIndexPair b) { + return (a.first > b.first); + } + }; + + typedef std::multiset<BleuIndexPair, HypothesisOrderer> HypoQueueType; + //typedef std::set<BleuIndexPair, HypothesisOrderer> HypoQueueType; + + HypoQueueType m_queue; + const size_t m_capacity; +}; + +} // namespace Moses diff --git a/mira/Jamfile b/mira/Jamfile new file mode 100644 index 000000000..cede96233 --- /dev/null +++ b/mira/Jamfile @@ -0,0 +1,13 @@ +lib mira_lib : +[ glob *.cpp : *Test.cpp Main.cpp ] +../moses-cmd/src//IOWrapper_lib ../mert//bleu_lib ../moses/src//moses ../OnDiskPt//OnDiskPt ..//boost_program_options ; + +exe mira : Main.cpp mira_lib ; + +alias programs : mira ; + +import testing ; + +unit-test mira_test : [ glob *Test.cpp ] mira_lib ..//boost_unit_test_framework ; + +explicit mira_test ;
\ No newline at end of file diff --git a/mira/Main.cpp b/mira/Main.cpp new file mode 100644 index 000000000..7fe332740 --- /dev/null +++ b/mira/Main.cpp @@ -0,0 +1,1990 @@ +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2010 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#include <algorithm> +#include <cstdlib> +#include <ctime> +#include <string> +#include <vector> +#include <map> + +#include <boost/program_options.hpp> +#include <boost/algorithm/string.hpp> + +#ifdef MPI_ENABLE +#include <boost/mpi.hpp> +namespace mpi = boost::mpi; +#endif + +#include "Main.h" +#include "FeatureVector.h" +#include "StaticData.h" +#include "ChartTrellisPathList.h" +#include "ChartTrellisPath.h" +#include "ScoreComponentCollection.h" +#include "Optimiser.h" +#include "Hildreth.h" +#include "ThreadPool.h" +#include "DummyScoreProducers.h" +#include "LexicalReordering.h" +#include "BleuScorer.h" +#include "HypothesisQueue.h" +#include "WordTranslationFeature.h" +#include "PhrasePairFeature.h" + +using namespace Mira; +using namespace std; +using namespace Moses; +namespace po = boost::program_options; + +int main(int argc, char** argv) { + size_t rank = 0; + size_t size = 1; +#ifdef MPI_ENABLE + mpi::environment env(argc,argv); + mpi::communicator world; + rank = world.rank(); + size = world.size(); +#endif + + bool help; + int verbosity; + string mosesConfigFile; + string inputFile; + vector<string> referenceFiles; + vector<string> mosesConfigFilesFolds, inputFilesFolds, referenceFilesFolds; + // string coreWeightFile, startWeightFile; + size_t epochs; + string learner; + bool shuffle; + size_t mixingFrequency; + size_t weightDumpFrequency; + string weightDumpStem; + bool scale_margin, scale_margin_precision; + bool scale_update, scale_update_precision; + size_t n; + size_t batchSize; + bool distinctNbest; + bool accumulateWeights; + float historySmoothing; + bool scaleByInputLength, scaleByAvgInputLength; + bool scaleByInverseLength, scaleByAvgInverseLength; + float scaleByX; + float slack; + bool averageWeights; + bool weightConvergence; + float learning_rate; + float mira_learning_rate; + float perceptron_learning_rate; + string decoder_settings; + float min_weight_change; + bool normaliseWeights, normaliseMargin; + bool print_feature_values; + bool historyBleu ; + bool sentenceBleu; + bool perceptron_update; + bool hope_fear; + bool model_hope_fear; + int hope_n, fear_n; + size_t bleu_smoothing_scheme; + float min_oracle_bleu; + float minBleuRatio, maxBleuRatio; + bool boost; + bool decode_hope, decode_fear, decode_model; + string decode_filename; + bool batchEqualsShard; + bool sparseAverage, dumpMixedWeights, sparseNoAverage; + int featureCutoff; + bool pruneZeroWeights; + bool printFeatureCounts, printNbestWithFeatures; + bool avgRefLength; + bool print_weights, print_core_weights, debug_model, scale_lm, scale_wp; + float scale_lm_factor, scale_wp_factor; + bool kbest; + string moses_src; + float sigmoidParam; + float bleuWeight, bleuWeight_hope, bleuWeight_fear; + bool bleu_weight_lm, bleu_weight_lm_adjust; + float bleu_weight_lm_factor; + bool l1_regularize, l2_regularize, l1_reg_sparse, l2_reg_sparse; + float l1_lambda, l2_lambda; + bool most_violated, most_violated_reg, all_violated, max_bleu_diff, one_against_all; + bool feature_confidence, signed_counts; + float decay_core, decay_sparse, core_r0, sparse_r0; + bool selective, summed; + float bleu_weight_fear_factor; + bool hildreth; + float add2lm; + bool realBleu, disableBleuFeature; + bool rescaleSlack; + bool makePairs; + bool debug; + bool reg_on_every_mix; + size_t continue_epoch; + bool modelPlusBleu, simpleHistoryBleu; + po::options_description desc("Allowed options"); + desc.add_options() + ("continue-epoch", po::value<size_t>(&continue_epoch)->default_value(0), "Continue an interrupted experiment from this epoch on") + ("freq-reg", po::value<bool>(®_on_every_mix)->default_value(false), "Regularize after every weight mixing") + ("l1sparse", po::value<bool>(&l1_reg_sparse)->default_value(true), "L1-regularization for sparse weights only") + ("l2sparse", po::value<bool>(&l2_reg_sparse)->default_value(true), "L2-regularization for sparse weights only") + ("mv-reg", po::value<bool>(&most_violated_reg)->default_value(false), "Regularize most violated constraint") + ("dbg", po::value<bool>(&debug)->default_value(true), "More debug output") + ("make-pairs", po::value<bool>(&makePairs)->default_value(true), "Make pairs of hypotheses for 1slack") + ("debug", po::value<bool>(&debug)->default_value(true), "More debug output") + ("rescale-slack", po::value<bool>(&rescaleSlack)->default_value(false), "Rescale slack in 1-slack formulation") + ("disable-bleu-feature", po::value<bool>(&disableBleuFeature)->default_value(false), "Disable the Bleu feature") + ("real-bleu", po::value<bool>(&realBleu)->default_value(false), "Compute real sentence Bleu on complete translations") + ("add2lm", po::value<float>(&add2lm)->default_value(0.0), "Add the specified amount to all LM weights") + ("hildreth", po::value<bool>(&hildreth)->default_value(false), "Prefer Hildreth over analytical update") + ("selective", po::value<bool>(&selective)->default_value(false), "Build constraints for every feature") + ("summed", po::value<bool>(&summed)->default_value(false), "Sum up all constraints") + ("model-plus-bleu", po::value<bool>(&modelPlusBleu)->default_value(false), "Use the sum of model score and +/- bleu to select hope and fear translations") + ("simple-history-bleu", po::value<bool>(&simpleHistoryBleu)->default_value(false), "Simple history Bleu") + + ("bleu-weight", po::value<float>(&bleuWeight)->default_value(1.0), "Bleu weight used in decoder objective") + ("bw-hope", po::value<float>(&bleuWeight_hope)->default_value(-1.0), "Bleu weight used in decoder objective for hope") + ("bw-fear", po::value<float>(&bleuWeight_fear)->default_value(-1.0), "Bleu weight used in decoder objective for fear") + + ("core-r0", po::value<float>(&core_r0)->default_value(1.0), "Start learning rate for core features") + ("sparse-r0", po::value<float>(&sparse_r0)->default_value(1.0), "Start learning rate for sparse features") + + ("tie-bw-to-lm", po::value<bool>(&bleu_weight_lm)->default_value(false), "Make bleu weight depend on lm weight") + ("adjust-bw", po::value<bool>(&bleu_weight_lm_adjust)->default_value(false), "Adjust bleu weight when lm weight changes") + ("bw-lm-factor", po::value<float>(&bleu_weight_lm_factor)->default_value(2.0), "Make bleu weight depend on lm weight by this factor") + ("bw-factor-fear", po::value<float>(&bleu_weight_fear_factor)->default_value(1.0), "Multiply fear weight by this factor") + ("accumulate-weights", po::value<bool>(&accumulateWeights)->default_value(false), "Accumulate and average weights over all epochs") + ("average-weights", po::value<bool>(&averageWeights)->default_value(false), "Set decoder weights to average weights after each update") + ("avg-ref-length", po::value<bool>(&avgRefLength)->default_value(false), "Use average reference length instead of shortest for BLEU score feature") + ("batch-equals-shard", po::value<bool>(&batchEqualsShard)->default_value(false), "Batch size is equal to shard size (purely batch)") + ("batch-size,b", po::value<size_t>(&batchSize)->default_value(1), "Size of batch that is send to optimiser for weight adjustments") + ("bleu-smoothing-scheme", po::value<size_t>(&bleu_smoothing_scheme)->default_value(1), "Set a smoothing scheme for sentence-Bleu: +1 (1), +0.1 (2), papineni (3) (default:1)") + ("boost", po::value<bool>(&boost)->default_value(false), "Apply boosting factor to updates on misranked candidates") + ("config,f", po::value<string>(&mosesConfigFile), "Moses ini-file") + ("configs-folds", po::value<vector<string> >(&mosesConfigFilesFolds), "Moses ini-files, one for each fold") + ("debug-model", po::value<bool>(&debug_model)->default_value(false), "Get best model translation for debugging purposes") + ("decode-hope", po::value<bool>(&decode_hope)->default_value(false), "Decode dev input set according to hope objective") + ("decode-fear", po::value<bool>(&decode_fear)->default_value(false), "Decode dev input set according to fear objective") + ("decode-model", po::value<bool>(&decode_model)->default_value(false), "Decode dev input set according to normal objective") + ("decode-filename", po::value<string>(&decode_filename), "Filename for Bleu objective translations") + ("decoder-settings", po::value<string>(&decoder_settings)->default_value(""), "Decoder settings for tuning runs") + ("distinct-nbest", po::value<bool>(&distinctNbest)->default_value(true), "Use n-best list with distinct translations in inference step") + ("dump-mixed-weights", po::value<bool>(&dumpMixedWeights)->default_value(false), "Dump mixed weights instead of averaged weights") + ("epochs,e", po::value<size_t>(&epochs)->default_value(10), "Number of epochs") + ("feature-cutoff", po::value<int>(&featureCutoff)->default_value(-1), "Feature cutoff as additional regularization for sparse features") + ("fear-n", po::value<int>(&fear_n)->default_value(1), "Number of fear translations used") + ("help", po::value(&help)->zero_tokens()->default_value(false), "Print this help message and exit") + ("history-bleu", po::value<bool>(&historyBleu)->default_value(false), "Use 1best translations to update the history") + ("history-smoothing", po::value<float>(&historySmoothing)->default_value(0.9), "Adjust the factor for history smoothing") + ("hope-fear", po::value<bool>(&hope_fear)->default_value(true), "Use only hope and fear translations for optimisation (not model)") + ("hope-n", po::value<int>(&hope_n)->default_value(2), "Number of hope translations used") + ("input-file,i", po::value<string>(&inputFile), "Input file containing tokenised source") + ("input-files-folds", po::value<vector<string> >(&inputFilesFolds), "Input files containing tokenised source, one for each fold") + ("learner,l", po::value<string>(&learner)->default_value("mira"), "Learning algorithm") + ("l1-lambda", po::value<float>(&l1_lambda)->default_value(0.0001), "Lambda for l1-regularization (w_i +/- lambda)") + ("l2-lambda", po::value<float>(&l2_lambda)->default_value(0.01), "Lambda for l2-regularization (w_i * (1 - lambda))") + ("l1-reg", po::value<bool>(&l1_regularize)->default_value(false), "L1-regularization") + ("l2-reg", po::value<bool>(&l2_regularize)->default_value(false), "L2-regularization") + ("min-bleu-ratio", po::value<float>(&minBleuRatio)->default_value(-1), "Set a minimum BLEU ratio between hope and fear") + ("max-bleu-ratio", po::value<float>(&maxBleuRatio)->default_value(-1), "Set a maximum BLEU ratio between hope and fear") + ("max-bleu-diff", po::value<bool>(&max_bleu_diff)->default_value(true), "Select hope/fear with maximum Bleu difference") + ("min-oracle-bleu", po::value<float>(&min_oracle_bleu)->default_value(0), "Set a minimum oracle BLEU score") + ("min-weight-change", po::value<float>(&min_weight_change)->default_value(0.0001), "Set minimum weight change for stopping criterion") + ("mira-learning-rate", po::value<float>(&mira_learning_rate)->default_value(1), "Learning rate for MIRA (fixed or flexible)") + ("mixing-frequency", po::value<size_t>(&mixingFrequency)->default_value(1), "How often per epoch to mix weights, when using mpi") + ("model-hope-fear", po::value<bool>(&model_hope_fear)->default_value(false), "Use model, hope and fear translations for optimisation") + ("moses-src", po::value<string>(&moses_src)->default_value(""), "Moses source directory") + ("nbest,n", po::value<size_t>(&n)->default_value(1), "Number of translations in n-best list") + ("normalise-weights", po::value<bool>(&normaliseWeights)->default_value(false), "Whether to normalise the updated weights before passing them to the decoder") + ("normalise-margin", po::value<bool>(&normaliseMargin)->default_value(false), "Normalise the margin: squash between 0 and 1") + ("perceptron-learning-rate", po::value<float>(&perceptron_learning_rate)->default_value(0.01), "Perceptron learning rate") + ("print-feature-values", po::value<bool>(&print_feature_values)->default_value(false), "Print out feature values") + ("print-feature-counts", po::value<bool>(&printFeatureCounts)->default_value(false), "Print out feature values, print feature list with hope counts after 1st epoch") + ("print-nbest-with-features", po::value<bool>(&printNbestWithFeatures)->default_value(false), "Print out feature values, print feature list with hope counts after 1st epoch") + ("print-weights", po::value<bool>(&print_weights)->default_value(false), "Print out current weights") + ("print-core-weights", po::value<bool>(&print_core_weights)->default_value(true), "Print out current core weights") + ("prune-zero-weights", po::value<bool>(&pruneZeroWeights)->default_value(false), "Prune zero-valued sparse feature weights") + ("reference-files,r", po::value<vector<string> >(&referenceFiles), "Reference translation files for training") + ("reference-files-folds", po::value<vector<string> >(&referenceFilesFolds), "Reference translation files for training, one for each fold") + ("kbest", po::value<bool>(&kbest)->default_value(false), "Select hope/fear pairs from a list of nbest translations") + + ("scale-by-inverse-length", po::value<bool>(&scaleByInverseLength)->default_value(false), "Scale BLEU by (history of) inverse input length") + ("scale-by-input-length", po::value<bool>(&scaleByInputLength)->default_value(false), "Scale BLEU by (history of) input length") + ("scale-by-avg-input-length", po::value<bool>(&scaleByAvgInputLength)->default_value(false), "Scale BLEU by average input length") + ("scale-by-avg-inverse-length", po::value<bool>(&scaleByAvgInverseLength)->default_value(false), "Scale BLEU by average inverse input length") + ("scale-by-x", po::value<float>(&scaleByX)->default_value(1), "Scale the BLEU score by value x") + ("scale-lm", po::value<bool>(&scale_lm)->default_value(false), "Scale the language model feature") + ("scale-factor-lm", po::value<float>(&scale_lm_factor)->default_value(2), "Scale the language model feature by this factor") + ("scale-wp", po::value<bool>(&scale_wp)->default_value(false), "Scale the word penalty feature") + ("scale-factor-wp", po::value<float>(&scale_wp_factor)->default_value(2), "Scale the word penalty feature by this factor") + ("scale-margin", po::value<bool>(&scale_margin)->default_value(0), "Scale the margin by the Bleu score of the oracle translation") + ("scale-margin-precision", po::value<bool>(&scale_margin_precision)->default_value(0), "Scale margin by precision of oracle") + ("scale-update", po::value<bool>(&scale_update)->default_value(0), "Scale update by Bleu score of oracle") + ("scale-update-precision", po::value<bool>(&scale_update_precision)->default_value(0), "Scale update by precision of oracle") + ("sentence-level-bleu", po::value<bool>(&sentenceBleu)->default_value(true), "Use a sentences level Bleu scoring function") + ("shuffle", po::value<bool>(&shuffle)->default_value(false), "Shuffle input sentences before processing") + ("sigmoid-param", po::value<float>(&sigmoidParam)->default_value(1), "y=sigmoidParam is the axis that this sigmoid approaches") + ("slack", po::value<float>(&slack)->default_value(0.01), "Use slack in optimiser") + ("sparse-average", po::value<bool>(&sparseAverage)->default_value(false), "Average weights by the number of processes") + ("sparse-no-average", po::value<bool>(&sparseNoAverage)->default_value(false), "Don't average sparse weights, just sum") + ("stop-weights", po::value<bool>(&weightConvergence)->default_value(true), "Stop when weights converge") + ("verbosity,v", po::value<int>(&verbosity)->default_value(0), "Verbosity level") + ("weight-dump-frequency", po::value<size_t>(&weightDumpFrequency)->default_value(1), "How often per epoch to dump weights (mpi)") + ("weight-dump-stem", po::value<string>(&weightDumpStem)->default_value("weights"), "Stem of filename to use for dumping weights"); + + po::options_description cmdline_options; + cmdline_options.add(desc); + po::variables_map vm; + po::store(po::command_line_parser(argc, argv). options(cmdline_options).run(), vm); + po::notify(vm); + + if (help) { + std::cout << "Usage: " + string(argv[0]) + + " -f mosesini-file -i input-file -r reference-file(s) [options]" << std::endl; + std::cout << desc << std::endl; + return 0; + } + + const StaticData &staticData = StaticData::Instance(); + + bool trainWithMultipleFolds = false; + if (mosesConfigFilesFolds.size() > 0 || inputFilesFolds.size() > 0 || referenceFilesFolds.size() > 0) { + if (rank == 0) + cerr << "Training with " << mosesConfigFilesFolds.size() << " folds" << endl; + trainWithMultipleFolds = true; + } + + if (dumpMixedWeights && (mixingFrequency != weightDumpFrequency)) { + cerr << "Set mixing frequency = weight dump frequency for dumping mixed weights!" << endl; + exit(1); + } + + if ((sparseAverage || sparseNoAverage) && averageWeights) { + cerr << "Parameters --sparse-average 1/--sparse-no-average 1 and --average-weights 1 are incompatible (not implemented)" << endl; + exit(1); + } + + if (trainWithMultipleFolds) { + if (!mosesConfigFilesFolds.size()) { + cerr << "Error: No moses ini files specified for training with folds" << endl; + exit(1); + } + + if (!inputFilesFolds.size()) { + cerr << "Error: No input files specified for training with folds" << endl; + exit(1); + } + + if (!referenceFilesFolds.size()) { + cerr << "Error: No reference files specified for training with folds" << endl; + exit(1); + } + } + else { + if (mosesConfigFile.empty()) { + cerr << "Error: No moses ini file specified" << endl; + return 1; + } + + if (inputFile.empty()) { + cerr << "Error: No input file specified" << endl; + return 1; + } + + if (!referenceFiles.size()) { + cerr << "Error: No reference files specified" << endl; + return 1; + } + } + + // load input and references + vector<string> inputSentences; + size_t inputSize = trainWithMultipleFolds? inputFilesFolds.size(): 0; + size_t refSize = trainWithMultipleFolds? referenceFilesFolds.size(): referenceFiles.size(); + vector<vector<string> > inputSentencesFolds(inputSize); + vector<vector<string> > referenceSentences(refSize); + + // number of cores for each fold + size_t coresPerFold = 0, myFold = 0; + if (trainWithMultipleFolds) { + if (mosesConfigFilesFolds.size() > size) { + cerr << "Number of cores has to be a multiple of the number of folds" << endl; + exit(1); + } + coresPerFold = size/mosesConfigFilesFolds.size(); + if (size % coresPerFold > 0) { + cerr << "Number of cores has to be a multiple of the number of folds" << endl; + exit(1); + } + + if (rank == 0) + cerr << "Number of cores per fold: " << coresPerFold << endl; + myFold = rank/coresPerFold; + cerr << "Rank " << rank << ", my fold: " << myFold << endl; + } + + // NOTE: we do not actually need the references here, because we are reading them in from StaticData + if (trainWithMultipleFolds) { + if (!loadSentences(inputFilesFolds[myFold], inputSentencesFolds[myFold])) { + cerr << "Error: Failed to load input sentences from " << inputFilesFolds[myFold] << endl; + exit(1); + } + VERBOSE(1, "Rank " << rank << " reading inputs from " << inputFilesFolds[myFold] << endl); + + if (!loadSentences(referenceFilesFolds[myFold], referenceSentences[myFold])) { + cerr << "Error: Failed to load reference sentences from " << referenceFilesFolds[myFold] << endl; + exit(1); + } + if (referenceSentences[myFold].size() != inputSentencesFolds[myFold].size()) { + cerr << "Error: Input file length (" << inputSentencesFolds[myFold].size() << ") != (" + << referenceSentences[myFold].size() << ") reference file length (rank " << rank << ")" << endl; + exit(1); + } + VERBOSE(1, "Rank " << rank << " reading references from " << referenceFilesFolds[myFold] << endl); + } + else { + if (!loadSentences(inputFile, inputSentences)) { + cerr << "Error: Failed to load input sentences from " << inputFile << endl; + return 1; + } + + for (size_t i = 0; i < referenceFiles.size(); ++i) { + if (!loadSentences(referenceFiles[i], referenceSentences[i])) { + cerr << "Error: Failed to load reference sentences from " + << referenceFiles[i] << endl; + return 1; + } + if (referenceSentences[i].size() != inputSentences.size()) { + cerr << "Error: Input file length (" << inputSentences.size() << ") != (" + << referenceSentences[i].size() << ") length of reference file " << i + << endl; + return 1; + } + } + } + + if (scaleByAvgInputLength || scaleByInverseLength || scaleByAvgInverseLength) + scaleByInputLength = false; + + if (historyBleu || simpleHistoryBleu) { + sentenceBleu = false; + cerr << "Using history Bleu. " << endl; + } + + if (kbest) { + realBleu = true; + disableBleuFeature = true; + cerr << "Use kbest lists and real Bleu scores, disable Bleu feature.." << endl; + } + + // initialise Moses + // add references to initialize Bleu feature + boost::trim(decoder_settings); + decoder_settings += " -mira -distinct-nbest -references"; + if (trainWithMultipleFolds) { + decoder_settings += " "; + decoder_settings += referenceFilesFolds[myFold]; + } + else { + for (size_t i=0; i < referenceFiles.size(); ++i) { + decoder_settings += " "; + decoder_settings += referenceFiles[i]; + } + } + + vector<string> decoder_params; + boost::split(decoder_params, decoder_settings, boost::is_any_of("\t ")); + + string configFile = trainWithMultipleFolds? mosesConfigFilesFolds[myFold] : mosesConfigFile; + VERBOSE(1, "Rank " << rank << " reading config file from " << configFile << endl); + MosesDecoder* decoder = new MosesDecoder(configFile, verbosity, decoder_params.size(), decoder_params); + decoder->setBleuParameters(disableBleuFeature, sentenceBleu, scaleByInputLength, scaleByAvgInputLength, + scaleByInverseLength, scaleByAvgInverseLength, + scaleByX, historySmoothing, bleu_smoothing_scheme, simpleHistoryBleu); + SearchAlgorithm searchAlgorithm = staticData.GetSearchAlgorithm(); + bool chartDecoding = (searchAlgorithm == ChartDecoding); + + // Optionally shuffle the sentences + vector<size_t> order; + if (trainWithMultipleFolds) { + for (size_t i = 0; i < inputSentencesFolds[myFold].size(); ++i) { + order.push_back(i); + } + } + else { + if (rank == 0) { + for (size_t i = 0; i < inputSentences.size(); ++i) { + order.push_back(i); + } + } + } + + // initialise optimizer + Optimiser* optimiser = NULL; + if (learner == "mira") { + if (rank == 0) { + cerr << "Optimising using Mira" << endl; + cerr << "slack: " << slack << ", learning rate: " << mira_learning_rate << endl; + cerr << "selective: " << selective << endl; + if (normaliseMargin) + cerr << "sigmoid parameter: " << sigmoidParam << endl; + } + optimiser = new MiraOptimiser(slack, scale_margin, scale_margin_precision, + scale_update, scale_update_precision, boost, normaliseMargin, sigmoidParam); + learning_rate = mira_learning_rate; + perceptron_update = false; + } else if (learner == "perceptron") { + if (rank == 0) { + cerr << "Optimising using Perceptron" << endl; + } + optimiser = new Perceptron(); + learning_rate = perceptron_learning_rate; + perceptron_update = true; + model_hope_fear = false; // mira only + hope_fear = false; // mira only + n = 1; + hope_n = 1; + fear_n = 1; + } else { + cerr << "Error: Unknown optimiser: " << learner << endl; + return 1; + } + + // resolve parameter dependencies + if (batchSize > 1 && perceptron_update) { + batchSize = 1; + cerr << "Info: Setting batch size to 1 for perceptron update" << endl; + } + + if (hope_n == -1) + hope_n = n; + if (fear_n == -1) + fear_n = n; + + if (model_hope_fear || kbest) + hope_fear = false; // is true by default + if (learner == "mira" && !(hope_fear || model_hope_fear || kbest)) { + cerr << "Error: Need to select one of parameters --hope-fear/--model-hope-fear/--kbest for mira update." << endl; + return 1; + } + +#ifdef MPI_ENABLE + if (!trainWithMultipleFolds) + mpi::broadcast(world, order, 0); +#endif + + // Create shards according to the number of processes used + vector<size_t> shard; + if (trainWithMultipleFolds) { + size_t shardSize = order.size()/coresPerFold; + size_t shardStart = (size_t) (shardSize * (rank % coresPerFold)); + size_t shardEnd = shardStart + shardSize; + if (rank % coresPerFold == coresPerFold - 1) { // last rank of each fold + shardEnd = order.size(); + shardSize = shardEnd - shardStart; + } + VERBOSE(1, "Rank: " << rank << ", shard size: " << shardSize << endl); + VERBOSE(1, "Rank: " << rank << ", shard start: " << shardStart << " shard end: " << shardEnd << endl); + shard.resize(shardSize); + copy(order.begin() + shardStart, order.begin() + shardEnd, shard.begin()); + batchSize = 1; + } + else { + size_t shardSize = order.size() / size; + size_t shardStart = (size_t) (shardSize * rank); + size_t shardEnd = (size_t) (shardSize * (rank + 1)); + if (rank == size - 1) { + shardEnd = order.size(); + shardSize = shardEnd - shardStart; + } + VERBOSE(1, "Rank: " << rank << " Shard size: " << shardSize << endl); + VERBOSE(1, "Rank: " << rank << " Shard start: " << shardStart << " Shard end: " << shardEnd << endl); + shard.resize(shardSize); + copy(order.begin() + shardStart, order.begin() + shardEnd, shard.begin()); + if (batchEqualsShard) + batchSize = shardSize; + } + + // get reference to feature functions + const vector<const ScoreProducer*> featureFunctions = + staticData.GetTranslationSystem(TranslationSystem::DEFAULT).GetFeatureFunctions(); + ScoreComponentCollection initialWeights = decoder->getWeights(); + + bool tuneMetaFeature = false; + const vector<const FeatureFunction*>& sparseProducers = staticData.GetTranslationSystem(TranslationSystem::DEFAULT).GetSparseProducers(); + for (unsigned i = 0; i < sparseProducers.size(); ++i) { + float spWeight = sparseProducers[i]->GetSparseProducerWeight(); + if (spWeight != 1.0) { + tuneMetaFeature = true; + cerr << "Rank " << rank << ", sparse Producer " << + sparseProducers[i]->GetScoreProducerWeightShortName() + << " weight: " << spWeight << endl; + } + } + + if (add2lm != 0) { + const LMList& lmList_new = staticData.GetLMList(); + for (LMList::const_iterator i = lmList_new.begin(); i != lmList_new.end(); ++i) { + float lmWeight = initialWeights.GetScoreForProducer(*i) + add2lm; + initialWeights.Assign(*i, lmWeight); + cerr << "Rank " << rank << ", add " << add2lm << " to lm weight." << endl; + } + } + + if (normaliseWeights) { + initialWeights.L1Normalise(); + cerr << "Rank " << rank << ", normalised initial weights: " << initialWeights << endl; + } + + decoder->setWeights(initialWeights); + + // set bleu weight to twice the size of the language model weight(s) + const LMList& lmList = staticData.GetLMList(); + if (bleu_weight_lm) { + float lmSum = 0; + for (LMList::const_iterator i = lmList.begin(); i != lmList.end(); ++i) + lmSum += abs(initialWeights.GetScoreForProducer(*i)); + bleuWeight = lmSum * bleu_weight_lm_factor; + cerr << "Set bleu weight to lm weight * " << bleu_weight_lm_factor << endl; + } + + if (bleuWeight_hope == -1) { + bleuWeight_hope = bleuWeight; + } + if (bleuWeight_fear == -1) { + bleuWeight_fear = bleuWeight; + } + bleuWeight_fear *= bleu_weight_fear_factor; + cerr << "Bleu weight: " << bleuWeight << endl; + cerr << "Bleu weight fear: " << bleuWeight_fear << endl; + + if (decode_hope || decode_fear || decode_model) { + size_t decode = 1; + if (decode_fear) decode = 2; + if (decode_model) decode = 3; + decodeHopeOrFear(rank, size, decode, decode_filename, inputSentences, decoder, n, bleuWeight); + } + + //Main loop: + ScoreComponentCollection cumulativeWeights; // collect weights per epoch to produce an average + ScoreComponentCollection cumulativeWeightsBinary; + size_t numberOfUpdates = 0; + size_t numberOfUpdatesThisEpoch = 0; + + time_t now; + time(&now); + cerr << "Rank " << rank << ", " << ctime(&now); + + float avgInputLength = 0; + float sumOfInputs = 0; + size_t numberOfInputs = 0; + + ScoreComponentCollection mixedWeights; + ScoreComponentCollection mixedWeightsPrevious; + ScoreComponentCollection mixedWeightsBeforePrevious; + ScoreComponentCollection mixedAverageWeights; + ScoreComponentCollection mixedAverageWeightsPrevious; + ScoreComponentCollection mixedAverageWeightsBeforePrevious; + + bool stop = false; +// int sumStillViolatedConstraints; + float epsilon = 0.0001; + + // Variables for feature confidence + ScoreComponentCollection confidenceCounts, mixedConfidenceCounts, featureLearningRates; + featureLearningRates.UpdateLearningRates(decay_core, decay_sparse, confidenceCounts, core_r0, sparse_r0); //initialise core learning rates + cerr << "Initial learning rates, core: " << core_r0 << ", sparse: " << sparse_r0 << endl; + + for (size_t epoch = continue_epoch; epoch < epochs && !stop; ++epoch) { + if (shuffle) { + if (trainWithMultipleFolds || rank == 0) { + cerr << "Rank " << rank << ", epoch " << epoch << ", shuffling input sentences.." << endl; + RandomIndex rindex; + random_shuffle(order.begin(), order.end(), rindex); + } + +#ifdef MPI_ENABLE + if (!trainWithMultipleFolds) + mpi::broadcast(world, order, 0); +#endif + + // redo shards + if (trainWithMultipleFolds) { + size_t shardSize = order.size()/coresPerFold; + size_t shardStart = (size_t) (shardSize * (rank % coresPerFold)); + size_t shardEnd = shardStart + shardSize; + if (rank % coresPerFold == coresPerFold - 1) { // last rank of each fold + shardEnd = order.size(); + shardSize = shardEnd - shardStart; + } + VERBOSE(1, "Rank: " << rank << ", shard size: " << shardSize << endl); + VERBOSE(1, "Rank: " << rank << ", shard start: " << shardStart << " shard end: " << shardEnd << endl); + shard.resize(shardSize); + copy(order.begin() + shardStart, order.begin() + shardEnd, shard.begin()); + batchSize = 1; + } + else { + size_t shardSize = order.size()/size; + size_t shardStart = (size_t) (shardSize * rank); + size_t shardEnd = (size_t) (shardSize * (rank + 1)); + if (rank == size - 1) { + shardEnd = order.size(); + shardSize = shardEnd - shardStart; + } + VERBOSE(1, "Shard size: " << shardSize << endl); + VERBOSE(1, "Rank: " << rank << " Shard start: " << shardStart << " Shard end: " << shardEnd << endl); + shard.resize(shardSize); + copy(order.begin() + shardStart, order.begin() + shardEnd, shard.begin()); + if (batchEqualsShard) + batchSize = shardSize; + } + } + + // sum of violated constraints in an epoch + // sumStillViolatedConstraints = 0; + + numberOfUpdatesThisEpoch = 0; + // Sum up weights over one epoch, final average uses weights from last epoch + if (!accumulateWeights) { + cumulativeWeights.ZeroAll(); + cumulativeWeightsBinary.ZeroAll(); + } + + // number of weight dumps this epoch + size_t weightMixingThisEpoch = 0; + size_t weightEpochDump = 0; + + size_t shardPosition = 0; + vector<size_t>::const_iterator sid = shard.begin(); + while (sid != shard.end()) { + // feature values for hypotheses i,j (matrix: batchSize x 3*n x featureValues) + vector<vector<ScoreComponentCollection> > featureValues; + vector<vector<float> > bleuScores; + vector<vector<float> > modelScores; + + // variables for hope-fear/perceptron setting + vector<vector<ScoreComponentCollection> > featureValuesHope; + vector<vector<ScoreComponentCollection> > featureValuesFear; + vector<vector<float> > bleuScoresHope; + vector<vector<float> > bleuScoresFear; + vector<vector<float> > modelScoresHope; + vector<vector<float> > modelScoresFear; + + // get moses weights + ScoreComponentCollection mosesWeights = decoder->getWeights(); + VERBOSE(1, "\nRank " << rank << ", epoch " << epoch << ", weights: " << mosesWeights << endl); + + if (historyBleu || simpleHistoryBleu) { + decoder->printBleuFeatureHistory(cerr); + } + + if (tuneMetaFeature) { + // initialise meta feature + MetaFeatureProducer *m = staticData.GetMetaFeatureProducer(); + FeatureFunction* ff = const_cast<FeatureFunction*>(sparseProducers[0]); + if (sparseProducers[0]->GetScoreProducerWeightShortName().compare("wt") == 0) { + WordTranslationFeature* wt = + static_cast<WordTranslationFeature*>(ff); + mosesWeights.Assign(m, wt->GetSparseProducerWeight()); + } + else if (sparseProducers[0]->GetScoreProducerWeightShortName().compare("pp") == 0) { + PhrasePairFeature* pp = + static_cast<PhrasePairFeature*>(ff); + mosesWeights.Assign(m, pp->GetSparseProducerWeight()); + } + } + + // BATCHING: produce nbest lists for all input sentences in batch + vector<float> oracleBleuScores; + vector<float> oracleModelScores; + vector<vector<const Word*> > oneBests; + vector<ScoreComponentCollection> oracleFeatureValues; + vector<size_t> inputLengths; + vector<size_t> ref_ids; + size_t actualBatchSize = 0; + + vector<size_t>::const_iterator current_sid_start = sid; + size_t examples_in_batch = 0; + bool skip_example = false; + for (size_t batchPosition = 0; batchPosition < batchSize && sid + != shard.end(); ++batchPosition) { + string input; + if (trainWithMultipleFolds) + input = inputSentencesFolds[myFold][*sid]; + else + input = inputSentences[*sid]; + + Moses::Sentence *sentence = new Sentence(); + stringstream in(input + "\n"); + const vector<FactorType> inputFactorOrder = staticData.GetInputFactorOrder(); + sentence->Read(in,inputFactorOrder); + cerr << "\nRank " << rank << ", epoch " << epoch << ", input sentence " << *sid << ": \""; + sentence->Print(cerr); + cerr << "\"" << " (batch pos " << batchPosition << ")" << endl; + size_t current_input_length = (*sentence).GetSize(); + + if (epoch == 0 && (scaleByAvgInputLength || scaleByAvgInverseLength)) { + sumOfInputs += current_input_length; + ++numberOfInputs; + avgInputLength = sumOfInputs/numberOfInputs; + decoder->setAvgInputLength(avgInputLength); + cerr << "Rank " << rank << ", epoch 0, average input length: " << avgInputLength << endl; + } + + vector<ScoreComponentCollection> newFeatureValues; + vector<float> newScores; + if (model_hope_fear) { + featureValues.push_back(newFeatureValues); + bleuScores.push_back(newScores); + modelScores.push_back(newScores); + } + if (hope_fear || perceptron_update) { + featureValuesHope.push_back(newFeatureValues); + featureValuesFear.push_back(newFeatureValues); + bleuScoresHope.push_back(newScores); + bleuScoresFear.push_back(newScores); + modelScoresHope.push_back(newScores); + modelScoresFear.push_back(newScores); + if (historyBleu || simpleHistoryBleu || debug_model) { + featureValues.push_back(newFeatureValues); + bleuScores.push_back(newScores); + modelScores.push_back(newScores); + } + } + if (kbest) { + // for decoding + featureValues.push_back(newFeatureValues); + bleuScores.push_back(newScores); + modelScores.push_back(newScores); + + // for storing selected examples + featureValuesHope.push_back(newFeatureValues); + featureValuesFear.push_back(newFeatureValues); + bleuScoresHope.push_back(newScores); + bleuScoresFear.push_back(newScores); + modelScoresHope.push_back(newScores); + modelScoresFear.push_back(newScores); + } + + size_t ref_length; + float avg_ref_length; + + if (print_weights) + cerr << "Rank " << rank << ", epoch " << epoch << ", current weights: " << mosesWeights << endl; + if (print_core_weights) { + cerr << "Rank " << rank << ", epoch " << epoch << ", current weights: "; + mosesWeights.PrintCoreFeatures(); + cerr << endl; + } + + // check LM weight + const LMList& lmList_new = staticData.GetLMList(); + for (LMList::const_iterator i = lmList_new.begin(); i != lmList_new.end(); ++i) { + float lmWeight = mosesWeights.GetScoreForProducer(*i); + cerr << "Rank " << rank << ", epoch " << epoch << ", lm weight: " << lmWeight << endl; + if (lmWeight <= 0) { + cerr << "Rank " << rank << ", epoch " << epoch << ", ERROR: language model weight should never be <= 0." << endl; + mosesWeights.Assign(*i, 0.1); + cerr << "Rank " << rank << ", epoch " << epoch << ", assign lm weights of 0.1" << endl; + } + } + + // select inference scheme + cerr << "Rank " << rank << ", epoch " << epoch << ", real Bleu? " << realBleu << endl; + if (hope_fear || perceptron_update) { + // HOPE + cerr << "Rank " << rank << ", epoch " << epoch << ", " << hope_n << + "best hope translations" << endl; + vector< vector<const Word*> > outputHope = decoder->getNBest(input, *sid, hope_n, 1.0, bleuWeight_hope, + featureValuesHope[batchPosition], bleuScoresHope[batchPosition], modelScoresHope[batchPosition], + 1, realBleu, distinctNbest, avgRefLength, rank, epoch, ""); + vector<const Word*> oracle = outputHope[0]; + decoder->cleanup(chartDecoding); + ref_length = decoder->getClosestReferenceLength(*sid, oracle.size()); + avg_ref_length = ref_length; + float hope_length_ratio = (float)oracle.size()/ref_length; + int oracleSize = (int)oracle.size(); + cerr << endl; + + // count sparse features occurring in hope translation + featureValuesHope[batchPosition][0].IncrementSparseHopeFeatures(); + + float precision = bleuScoresHope[batchPosition][0]; + if (historyBleu || simpleHistoryBleu) { + precision /= decoder->getTargetLengthHistory(); + } + else { + if (scaleByAvgInputLength) precision /= decoder->getAverageInputLength(); + else if (scaleByAvgInverseLength) precision /= (100/decoder->getAverageInputLength()); + precision /= scaleByX; + } + if (scale_margin_precision || scale_update_precision) { + if (historyBleu || simpleHistoryBleu || scaleByAvgInputLength || scaleByAvgInverseLength) { + cerr << "Rank " << rank << ", epoch " << epoch << ", set hope precision: " << precision << endl; + ((MiraOptimiser*) optimiser)->setPrecision(precision); + } + } + + vector<const Word*> bestModel; + if (debug_model || historyBleu || simpleHistoryBleu) { + // MODEL (for updating the history only, using dummy vectors) + cerr << "Rank " << rank << ", epoch " << epoch << ", 1best wrt model score (debug or history)" << endl; + vector< vector<const Word*> > outputModel = decoder->getNBest(input, *sid, n, 0.0, bleuWeight, + featureValues[batchPosition], bleuScores[batchPosition], modelScores[batchPosition], + 1, realBleu, distinctNbest, avgRefLength, rank, epoch, ""); + bestModel = outputModel[0]; + decoder->cleanup(chartDecoding); + cerr << endl; + ref_length = decoder->getClosestReferenceLength(*sid, bestModel.size()); + } + + // FEAR + float fear_length_ratio = 0; + float bleuRatioHopeFear = 0; + int fearSize = 0; + cerr << "Rank " << rank << ", epoch " << epoch << ", " << fear_n << "best fear translations" << endl; + vector< vector<const Word*> > outputFear = decoder->getNBest(input, *sid, fear_n, -1.0, bleuWeight_fear, + featureValuesFear[batchPosition], bleuScoresFear[batchPosition], modelScoresFear[batchPosition], + 1, realBleu, distinctNbest, avgRefLength, rank, epoch, ""); + vector<const Word*> fear = outputFear[0]; + decoder->cleanup(chartDecoding); + ref_length = decoder->getClosestReferenceLength(*sid, fear.size()); + avg_ref_length += ref_length; + avg_ref_length /= 2; + fear_length_ratio = (float)fear.size()/ref_length; + fearSize = (int)fear.size(); + cerr << endl; + for (size_t i = 0; i < fear.size(); ++i) + delete fear[i]; + + // count sparse features occurring in fear translation + featureValuesFear[batchPosition][0].IncrementSparseFearFeatures(); + + // Bleu-related example selection + bool skip = false; + bleuRatioHopeFear = bleuScoresHope[batchPosition][0] / bleuScoresFear[batchPosition][0]; + if (minBleuRatio != -1 && bleuRatioHopeFear < minBleuRatio) + skip = true; + if(maxBleuRatio != -1 && bleuRatioHopeFear > maxBleuRatio) + skip = true; + + // sanity check + if (historyBleu || simpleHistoryBleu) { + if (bleuScores[batchPosition][0] > bleuScoresHope[batchPosition][0] && + modelScores[batchPosition][0] > modelScoresHope[batchPosition][0]) { + if (abs(bleuScores[batchPosition][0] - bleuScoresHope[batchPosition][0]) > epsilon && + abs(modelScores[batchPosition][0] - modelScoresHope[batchPosition][0]) > epsilon) { + cerr << "Rank " << rank << ", epoch " << epoch << ", ERROR: MODEL translation better than HOPE translation." << endl; + skip = true; + } + } + if (bleuScoresFear[batchPosition][0] > bleuScores[batchPosition][0] && + modelScoresFear[batchPosition][0] > modelScores[batchPosition][0]) { + if (abs(bleuScoresFear[batchPosition][0] - bleuScores[batchPosition][0]) > epsilon && + abs(modelScoresFear[batchPosition][0] - modelScores[batchPosition][0]) > epsilon) { + cerr << "Rank " << rank << ", epoch " << epoch << ", ERROR: FEAR translation better than MODEL translation." << endl; + skip = true; + } + } + } + if (bleuScoresFear[batchPosition][0] > bleuScoresHope[batchPosition][0]) { + if (abs(bleuScoresFear[batchPosition][0] - bleuScoresHope[batchPosition][0]) > epsilon) { + // check if it's an error or a warning + skip = true; + if (modelScoresFear[batchPosition][0] > modelScoresHope[batchPosition][0] && abs(modelScoresFear[batchPosition][0] - modelScoresHope[batchPosition][0]) > epsilon) { + cerr << "Rank " << rank << ", epoch " << epoch << ", ERROR: FEAR translation better than HOPE translation. (abs-diff: " << abs(bleuScoresFear[batchPosition][0] - bleuScoresHope[batchPosition][0]) << ")" <<endl; + } + else { + cerr << "Rank " << rank << ", epoch " << epoch << ", WARNING: FEAR translation has better Bleu than HOPE translation. (abs-diff: " << abs(bleuScoresFear[batchPosition][0] - bleuScoresHope[batchPosition][0]) << ")" <<endl; + } + } + } + + if (skip) { + cerr << "Rank " << rank << ", epoch " << epoch << ", skip example (" << hope_length_ratio << ", " << bleuRatioHopeFear << ").. " << endl; + featureValuesHope[batchPosition].clear(); + featureValuesFear[batchPosition].clear(); + bleuScoresHope[batchPosition].clear(); + bleuScoresFear[batchPosition].clear(); + if (historyBleu || simpleHistoryBleu || debug_model) { + featureValues[batchPosition].clear(); + bleuScores[batchPosition].clear(); + } + } + else { + examples_in_batch++; + + // needed for history + if (historyBleu || simpleHistoryBleu) { + inputLengths.push_back(current_input_length); + ref_ids.push_back(*sid); + oneBests.push_back(bestModel); + } + } + } + if (model_hope_fear) { + cerr << "Rank " << rank << ", epoch " << epoch << ", " << n << "best hope translations" << endl; + size_t oraclePos = featureValues[batchPosition].size(); + decoder->getNBest(input, *sid, n, 1.0, bleuWeight_hope, + featureValues[batchPosition], bleuScores[batchPosition], modelScores[batchPosition], + 0, realBleu, distinctNbest, avgRefLength, rank, epoch, ""); + //vector<const Word*> oracle = outputHope[0]; + // needed for history + inputLengths.push_back(current_input_length); + ref_ids.push_back(*sid); + decoder->cleanup(chartDecoding); + //ref_length = decoder->getClosestReferenceLength(*sid, oracle.size()); + //float hope_length_ratio = (float)oracle.size()/ref_length; + cerr << endl; + + oracleFeatureValues.push_back(featureValues[batchPosition][oraclePos]); + oracleBleuScores.push_back(bleuScores[batchPosition][oraclePos]); + oracleModelScores.push_back(modelScores[batchPosition][oraclePos]); + + // MODEL + cerr << "Rank " << rank << ", epoch " << epoch << ", " << n << "best wrt model score" << endl; + if (historyBleu || simpleHistoryBleu) { + vector< vector<const Word*> > outputModel = decoder->getNBest(input, *sid, n, 0.0, + bleuWeight, featureValues[batchPosition], bleuScores[batchPosition], + modelScores[batchPosition], 1, realBleu, distinctNbest, avgRefLength, rank, epoch, ""); + vector<const Word*> bestModel = outputModel[0]; + oneBests.push_back(bestModel); + inputLengths.push_back(current_input_length); + ref_ids.push_back(*sid); + } + else { + decoder->getNBest(input, *sid, n, 0.0, bleuWeight, + featureValues[batchPosition], bleuScores[batchPosition], modelScores[batchPosition], + 0, realBleu, distinctNbest, avgRefLength, rank, epoch, ""); + } + decoder->cleanup(chartDecoding); + //ref_length = decoder->getClosestReferenceLength(*sid, bestModel.size()); + //float model_length_ratio = (float)bestModel.size()/ref_length; + cerr << endl; + + // FEAR + cerr << "Rank " << rank << ", epoch " << epoch << ", " << n << "best fear translations" << endl; + decoder->getNBest(input, *sid, n, -1.0, bleuWeight_fear, + featureValues[batchPosition], bleuScores[batchPosition], modelScores[batchPosition], + 0, realBleu, distinctNbest, avgRefLength, rank, epoch, ""); + decoder->cleanup(chartDecoding); + //ref_length = decoder->getClosestReferenceLength(*sid, fear.size()); + //float fear_length_ratio = (float)fear.size()/ref_length; + + examples_in_batch++; + } + if (kbest) { + // MODEL + cerr << "Rank " << rank << ", epoch " << epoch << ", " << n << "best wrt model score" << endl; + if (historyBleu || simpleHistoryBleu) { + vector< vector<const Word*> > outputModel = decoder->getNBest(input, *sid, n, 0.0, + bleuWeight, featureValues[batchPosition], bleuScores[batchPosition], + modelScores[batchPosition], 1, realBleu, distinctNbest, avgRefLength, rank, epoch, ""); + vector<const Word*> bestModel = outputModel[0]; + oneBests.push_back(bestModel); + inputLengths.push_back(current_input_length); + ref_ids.push_back(*sid); + } + else { + decoder->getNBest(input, *sid, n, 0.0, bleuWeight, + featureValues[batchPosition], bleuScores[batchPosition], + modelScores[batchPosition], 0, realBleu, distinctNbest, avgRefLength, rank, epoch, ""); + } + decoder->cleanup(chartDecoding); + //ref_length = decoder->getClosestReferenceLength(*sid, bestModel.size()); + //float model_length_ratio = (float)bestModel.size()/ref_length; + cerr << endl; + + examples_in_batch++; + + HypothesisQueue queueHope(hope_n); + HypothesisQueue queueFear(fear_n); + cerr << endl; + if (most_violated || all_violated || one_against_all) { + float bleuHope = -1000; + float bleuFear = 1000; + size_t indexHope = -1; + size_t indexFear = -1; + + vector<float> bleuHopeList; + vector<float> bleuFearList; + vector<float> indexHopeList; + vector<float> indexFearList; + + if (most_violated) + cerr << "Rank " << rank << ", epoch " << epoch << ", pick pair with most violated constraint" << endl; + else if (all_violated) + cerr << "Rank " << rank << ", epoch " << epoch << ", pick all pairs with violated constraints"; + else + cerr << "Rank " << rank << ", epoch " << epoch << ", pick all pairs with hope"; + + // find best hope, then find fear that violates our constraint most + for (size_t i=0; i<bleuScores[batchPosition].size(); ++i) { + if (abs(bleuScores[batchPosition][i] - bleuHope) < epsilon) { // equal bleu scores + if (modelScores[batchPosition][i] > modelScores[batchPosition][indexHope]) { + if (abs(modelScores[batchPosition][i] - modelScores[batchPosition][indexHope]) > epsilon) { + // better model score + bleuHope = bleuScores[batchPosition][i]; + indexHope = i; + } + } + } + else if (bleuScores[batchPosition][i] > bleuHope) { // better than current best + bleuHope = bleuScores[batchPosition][i]; + indexHope = i; + } + } + + float currentViolation = 0; + float minimum_bleu_diff = 0.01; + for (size_t i=0; i<bleuScores[batchPosition].size(); ++i) { + float bleuDiff = bleuHope - bleuScores[batchPosition][i]; + float modelDiff = modelScores[batchPosition][indexHope] - modelScores[batchPosition][i]; + if (bleuDiff > epsilon) { + if (one_against_all && bleuDiff > minimum_bleu_diff) { + cerr << ".. adding pair"; + bleuHopeList.push_back(bleuHope); + bleuFearList.push_back(bleuScores[batchPosition][i]); + indexHopeList.push_back(indexHope); + indexFearList.push_back(i); + } + else if (modelDiff < bleuDiff) { + float diff = bleuDiff - modelDiff; + if (diff > epsilon) { + if (all_violated) { + cerr << ".. adding pair"; + bleuHopeList.push_back(bleuHope); + bleuFearList.push_back(bleuScores[batchPosition][i]); + indexHopeList.push_back(indexHope); + indexFearList.push_back(i); + } + else if (most_violated && diff > currentViolation) { + currentViolation = diff; + bleuFear = bleuScores[batchPosition][i]; + indexFear = i; + cerr << "Rank " << rank << ", epoch " << epoch << ", current violation: " << currentViolation << " (" << modelDiff << " >= " << bleuDiff << ")" << endl; + } + } + } + } + } + + if (most_violated) { + if (currentViolation > 0) { + cerr << "Rank " << rank << ", epoch " << epoch << ", adding pair with violation " << currentViolation << endl; + cerr << "Rank " << rank << ", epoch " << epoch << ", hope: " << bleuHope << " (" << indexHope << "), fear: " << bleuFear << " (" << indexFear << ")" << endl; + bleuScoresHope[batchPosition].push_back(bleuHope); + bleuScoresFear[batchPosition].push_back(bleuFear); + featureValuesHope[batchPosition].push_back(featureValues[batchPosition][indexHope]); + featureValuesFear[batchPosition].push_back(featureValues[batchPosition][indexFear]); + float modelScoreHope = modelScores[batchPosition][indexHope]; + float modelScoreFear = modelScores[batchPosition][indexFear]; + if (most_violated_reg) { + // reduce model score difference by factor ~0.5 + float reg = currentViolation/4; + modelScoreHope += abs(reg); + modelScoreFear -= abs(reg); + float newViolation = (bleuHope - bleuFear) - (modelScoreHope - modelScoreFear); + cerr << "Rank " << rank << ", epoch " << epoch << ", regularized violation: " << newViolation << endl; + } + modelScoresHope[batchPosition].push_back(modelScoreHope); + modelScoresFear[batchPosition].push_back(modelScoreFear); + + featureValues[batchPosition][indexHope].IncrementSparseHopeFeatures(); + featureValues[batchPosition][indexFear].IncrementSparseFearFeatures(); + } + else { + cerr << "Rank " << rank << ", epoch " << epoch << ", no violated constraint found." << endl; + skip_example = 1; + } + } + else cerr << endl; + } + if (max_bleu_diff) { + cerr << "Rank " << rank << ", epoch " << epoch << ", pick pair with max Bleu diff from list: " << bleuScores[batchPosition].size() << endl; + for (size_t i=0; i<bleuScores[batchPosition].size(); ++i) { + float hopeScore = bleuScores[batchPosition][i]; + if (modelPlusBleu) hopeScore += modelScores[batchPosition][i]; + BleuIndexPair hope(hopeScore, i); + queueHope.Push(hope); + + float fearScore = -1*(bleuScores[batchPosition][i]); + if (modelPlusBleu) fearScore += modelScores[batchPosition][i]; + BleuIndexPair fear(fearScore, i); + queueFear.Push(fear); + } + skip_example = 0; + } + cerr << endl; + + vector<BleuIndexPair> hopeList, fearList; + for (size_t i=0; i<hope_n && !queueHope.Empty(); ++i) hopeList.push_back(queueHope.Pop()); + for (size_t i=0; i<fear_n && !queueFear.Empty(); ++i) fearList.push_back(queueFear.Pop()); + for (size_t i=0; i<hopeList.size(); ++i) { + //float bleuHope = hopeList[i].first; + size_t indexHope = hopeList[i].second; + float bleuHope = bleuScores[batchPosition][indexHope]; + for (size_t j=0; j<fearList.size(); ++j) { + //float bleuFear = -1*(fearList[j].first); + size_t indexFear = fearList[j].second; + float bleuFear = bleuScores[batchPosition][indexFear]; + cerr << "Rank " << rank << ", epoch " << epoch << ", hope: " << bleuHope << " (" << indexHope << "), fear: " << bleuFear << " (" << indexFear << ")" << endl; + bleuScoresHope[batchPosition].push_back(bleuHope); + bleuScoresFear[batchPosition].push_back(bleuFear); + featureValuesHope[batchPosition].push_back(featureValues[batchPosition][indexHope]); + featureValuesFear[batchPosition].push_back(featureValues[batchPosition][indexFear]); + float modelScoreHope = modelScores[batchPosition][indexHope]; + float modelScoreFear = modelScores[batchPosition][indexFear]; + + modelScoresHope[batchPosition].push_back(modelScoreHope); + modelScoresFear[batchPosition].push_back(modelScoreFear); + + featureValues[batchPosition][indexHope].IncrementSparseHopeFeatures(); + featureValues[batchPosition][indexFear].IncrementSparseFearFeatures(); + } + } + if (!makePairs) + cerr << "Rank " << rank << ", epoch " << epoch << "summing up hope and fear vectors, no pairs" << endl; + } + + // next input sentence + ++sid; + ++actualBatchSize; + ++shardPosition; + } // end of batch loop + + if (examples_in_batch == 0 || (kbest && skip_example)) { + cerr << "Rank " << rank << ", epoch " << epoch << ", batch is empty." << endl; + } + else { + vector<vector<float> > losses(actualBatchSize); + if (model_hope_fear) { + // Set loss for each sentence as BLEU(oracle) - BLEU(hypothesis) + for (size_t batchPosition = 0; batchPosition < actualBatchSize; ++batchPosition) { + for (size_t j = 0; j < bleuScores[batchPosition].size(); ++j) { + losses[batchPosition].push_back(oracleBleuScores[batchPosition] - bleuScores[batchPosition][j]); + } + } + } + + // set weight for bleu feature to 0 before optimizing + vector<const ScoreProducer*>::const_iterator iter; + const vector<const ScoreProducer*> featureFunctions2 = staticData.GetTranslationSystem(TranslationSystem::DEFAULT).GetFeatureFunctions(); + for (iter = featureFunctions2.begin(); iter != featureFunctions2.end(); ++iter) { + if ((*iter)->GetScoreProducerWeightShortName() == "bl") { + mosesWeights.Assign(*iter, 0); + break; + } + } + + // scale LM feature (to avoid rapid changes) + if (scale_lm) { + cerr << "scale lm" << endl; + const LMList& lmList_new = staticData.GetLMList(); + for (LMList::const_iterator iter = lmList_new.begin(); iter != lmList_new.end(); ++iter) { + // scale down score + if (model_hope_fear) { + scaleFeatureScore(*iter, scale_lm_factor, featureValues, rank, epoch); + } + else { + scaleFeatureScore(*iter, scale_lm_factor, featureValuesHope, rank, epoch); + scaleFeatureScore(*iter, scale_lm_factor, featureValuesFear, rank, epoch); + } + } + } + + // scale WP + if (scale_wp) { + // scale up weight + WordPenaltyProducer *wp = staticData.GetFirstWordPenaltyProducer(); + + // scale down score + if (model_hope_fear) { + scaleFeatureScore(wp, scale_wp_factor, featureValues, rank, epoch); + } + else { + scaleFeatureScore(wp, scale_wp_factor, featureValuesHope, rank, epoch); + scaleFeatureScore(wp, scale_wp_factor, featureValuesFear, rank, epoch); + } + } + + // print out the feature values + if (print_feature_values) { + cerr << "\nRank " << rank << ", epoch " << epoch << ", feature values: " << endl; + if (model_hope_fear) printFeatureValues(featureValues); + else { + cerr << "hope: " << endl; + printFeatureValues(featureValuesHope); + cerr << "fear: " << endl; + printFeatureValues(featureValuesFear); + } + } + + // apply learning rates to feature vectors before optimization + if (feature_confidence) { + cerr << "Rank " << rank << ", epoch " << epoch << ", apply feature learning rates with decays " << decay_core << "/" << decay_sparse << ": " << featureLearningRates << endl; + if (model_hope_fear) { + applyPerFeatureLearningRates(featureValues, featureLearningRates, sparse_r0); + } + else { + applyPerFeatureLearningRates(featureValuesHope, featureLearningRates, sparse_r0); + applyPerFeatureLearningRates(featureValuesFear, featureLearningRates, sparse_r0); + } + } + else { + // apply fixed learning rates + cerr << "Rank " << rank << ", epoch " << epoch << ", apply fixed learning rates, core: " << core_r0 << ", sparse: " << sparse_r0 << endl; + if (core_r0 != 1.0 || sparse_r0 != 1.0) { + if (model_hope_fear) { + applyLearningRates(featureValues, core_r0, sparse_r0); + } + else { + applyLearningRates(featureValuesHope, core_r0, sparse_r0); + applyLearningRates(featureValuesFear, core_r0, sparse_r0); + } + } + } + + if (kbest) { + // If we are tuning a global weight for a sparse producer, + // we must collapse the sparse features first (report weighted aggregate) + if (tuneMetaFeature) { + for (unsigned i = 0; i < sparseProducers.size(); ++i) { + float spWeight = sparseProducers[i]->GetSparseProducerWeight(); + if (spWeight != 1.0) { + MetaFeatureProducer *m = staticData.GetMetaFeatureProducer(); + for (size_t i=0; i < featureValuesHope.size(); ++i) { + for (size_t j=0; j < featureValuesHope[i].size(); ++j) { + // multiply sparse feature values with weights + const FVector scores = + featureValuesHope[i][j].GetVectorForProducer(sparseProducers[i]); + const FVector &weights = staticData.GetAllWeights().GetScoresVector(); + float aggregate = scores.inner_product(weights); + //cerr << "Rank " << rank << ", epoch " << epoch << ", sparse Producer " << + //sparseProducers[i]->GetScoreProducerWeightShortName() + //<< " aggregate: " << aggregate << endl; + aggregate *= spWeight; + //cerr << "Rank " << rank << ", epoch " << epoch << ", sparse Producer " << + //sparseProducers[i]->GetScoreProducerWeightShortName() + //<< " weighted aggregate: " << aggregate << endl; + + // copy core features to a new collection, then assign aggregated sparse feature + ScoreComponentCollection scoresAggregate; + scoresAggregate.CoreAssign(featureValuesHope[i][j]); + scoresAggregate.Assign(m, aggregate); + featureValuesHope[i][j] = scoresAggregate; + } + } + for (size_t i=0; i < featureValuesFear.size(); ++i) { + for (size_t j=0; j < featureValuesFear[i].size(); ++j) { + // multiply sparse feature values with weights + const FVector scores = + featureValuesFear[i][j].GetVectorForProducer(sparseProducers[i]); + const FVector &weights = staticData.GetAllWeights().GetScoresVector(); + float aggregate = scores.inner_product(weights); + aggregate *= spWeight; + + // copy core features to a new collection, then assign aggregated sparse feature + ScoreComponentCollection scoresAggregate; + scoresAggregate.CoreAssign(featureValuesFear[i][j]); + scoresAggregate.Assign(m, aggregate); + featureValuesFear[i][j] = scoresAggregate; + } + } + + cerr << "Rank " << rank << ", epoch " << epoch << ", new hope feature vector: " << + featureValuesHope[0][0] << endl; + cerr << "Rank " << rank << ", epoch " << epoch << ", new fear feature vector: " << + featureValuesFear[0][0] << endl; + } + } + } + } + + // Run optimiser on batch: + VERBOSE(1, "\nRank " << rank << ", epoch " << epoch << ", run optimiser:" << endl); + size_t update_status = 1; + ScoreComponentCollection weightUpdate; + if (perceptron_update) { + vector<vector<float> > dummy1; + update_status = optimiser->updateWeightsHopeFear( weightUpdate, featureValuesHope, + featureValuesFear, dummy1, dummy1, dummy1, dummy1, learning_rate, rank, epoch); + } + else if (hope_fear) { + if (bleuScoresHope[0][0] >= min_oracle_bleu) { + if (hope_n == 1 && fear_n ==1 && batchSize == 1 && !hildreth) { + update_status = ((MiraOptimiser*) optimiser)->updateWeightsAnalytically(weightUpdate, + featureValuesHope[0][0], featureValuesFear[0][0], bleuScoresHope[0][0], + bleuScoresFear[0][0], modelScoresHope[0][0], modelScoresFear[0][0], learning_rate, rank, epoch); + } + else + update_status = optimiser->updateWeightsHopeFear(weightUpdate, featureValuesHope, + featureValuesFear, bleuScoresHope, bleuScoresFear, modelScoresHope, + modelScoresFear, learning_rate, rank, epoch); + } + else + update_status = 1; + } + else if (kbest) { + if (selective) + update_status = ((MiraOptimiser*)optimiser)->updateWeightsHopeFearSelective( + weightUpdate, featureValuesHope, featureValuesFear, bleuScoresHope, bleuScoresFear, + modelScoresHope, modelScoresFear, learning_rate, rank, epoch); + else if (summed) + update_status = ((MiraOptimiser*)optimiser)->updateWeightsHopeFearSummed( + weightUpdate, featureValuesHope, featureValuesFear, bleuScoresHope, bleuScoresFear, + modelScoresHope, modelScoresFear, learning_rate, rank, epoch, rescaleSlack, makePairs); + else { + if (batchSize == 1 && featureValuesHope[0].size() == 1 && !hildreth) { + cerr << "Rank " << rank << ", epoch " << epoch << ", model score hope: " << modelScoresHope[0][0] << endl; + cerr << "Rank " << rank << ", epoch " << epoch << ", model score fear: " << modelScoresFear[0][0] << endl; + update_status = ((MiraOptimiser*) optimiser)->updateWeightsAnalytically( + weightUpdate, featureValuesHope[0][0], featureValuesFear[0][0], + bleuScoresHope[0][0], bleuScoresFear[0][0], modelScoresHope[0][0], + modelScoresFear[0][0], learning_rate, rank, epoch); + } + else { + cerr << "Rank " << rank << ", epoch " << epoch << ", model score hope: " << modelScoresHope[0][0] << endl; + cerr << "Rank " << rank << ", epoch " << epoch << ", model score fear: " << modelScoresFear[0][0] << endl; + update_status = optimiser->updateWeightsHopeFear(weightUpdate, featureValuesHope, + featureValuesFear, bleuScoresHope, bleuScoresFear, modelScoresHope, + modelScoresFear, learning_rate, rank, epoch); + } + } + } + else { + // model_hope_fear + update_status = ((MiraOptimiser*) optimiser)->updateWeights(weightUpdate, + featureValues, losses, bleuScores, modelScores, oracleFeatureValues, + oracleBleuScores, oracleModelScores, learning_rate, rank, epoch); + } + + // sumStillViolatedConstraints += update_status; + + if (update_status == 0) { // if weights were updated + // apply weight update + if (debug) + cerr << "Rank " << rank << ", epoch " << epoch << ", update: " << weightUpdate << endl; + + if (tuneMetaFeature) { + MetaFeatureProducer *m = staticData.GetMetaFeatureProducer(); + // update sparse producer weight + // (NOTE: this currently doesn't work for more than one sparse producer) + float metaWeightUpdate = weightUpdate.GetScoreForProducer(m); + + const vector<const FeatureFunction*> sparseProducers = + staticData.GetTranslationSystem(TranslationSystem::DEFAULT).GetSparseProducers(); + FeatureFunction* ff = const_cast<FeatureFunction*>(sparseProducers[0]); + if (sparseProducers[0]->GetScoreProducerWeightShortName().compare("wt") == 0) { + WordTranslationFeature* wt = + static_cast<WordTranslationFeature*>(ff); + float newWeight = wt->GetSparseProducerWeight(); + cerr << "Rank " << rank << ", epoch " << epoch << ", old meta weight: " << newWeight << endl; + newWeight += metaWeightUpdate; + wt->SetSparseProducerWeight(newWeight); + cerr << "Rank " << rank << ", epoch " << epoch << ", new meta weight: " << newWeight << endl; + } + else if (sparseProducers[0]->GetScoreProducerWeightShortName().compare("pp") == 0) { + PhrasePairFeature* pp = + static_cast<PhrasePairFeature*>(ff); + float newWeight = pp->GetSparseProducerWeight(); + cerr << "Rank " << rank << ", epoch " << epoch << ", old meta weight: " << newWeight << endl; + newWeight += metaWeightUpdate; + pp->SetSparseProducerWeight(newWeight); + cerr << "Rank " << rank << ", epoch " << epoch << ", new meta weight: " << newWeight << endl; + } + } + + if (feature_confidence) { + // update confidence counts based on weight update + confidenceCounts.UpdateConfidenceCounts(weightUpdate, signed_counts); + + // update feature learning rates + featureLearningRates.UpdateLearningRates(decay_core, decay_sparse, confidenceCounts, core_r0, sparse_r0); + } + + // apply weight update to Moses weights + mosesWeights.PlusEquals(weightUpdate); + + if (normaliseWeights && !tuneMetaFeature) + mosesWeights.L1Normalise(); + + cumulativeWeights.PlusEquals(mosesWeights); + if (sparseAverage) { + ScoreComponentCollection binary; + binary.SetToBinaryOf(mosesWeights); + cumulativeWeightsBinary.PlusEquals(binary); + } + + ++numberOfUpdates; + ++numberOfUpdatesThisEpoch; + if (averageWeights && !tuneMetaFeature) { + ScoreComponentCollection averageWeights(cumulativeWeights); + if (accumulateWeights) { + averageWeights.DivideEquals(numberOfUpdates); + } else { + averageWeights.DivideEquals(numberOfUpdatesThisEpoch); + } + + mosesWeights = averageWeights; + } + + // set new Moses weights + decoder->setWeights(mosesWeights); + //cerr << "Rank " << rank << ", epoch " << epoch << ", new weights: " << mosesWeights << endl; + } + + // update history (for approximate document Bleu) + if (historyBleu || simpleHistoryBleu) { + for (size_t i = 0; i < oneBests.size(); ++i) + cerr << "Rank " << rank << ", epoch " << epoch << ", update history with 1best length: " << oneBests[i].size() << " "; + decoder->updateHistory(oneBests, inputLengths, ref_ids, rank, epoch); + deleteTranslations(oneBests); + } + } // END TRANSLATE AND UPDATE BATCH + + // size of all shards except for the last one + size_t generalShardSize; + if (trainWithMultipleFolds) + generalShardSize = order.size()/coresPerFold; + else + generalShardSize = order.size()/size; + + size_t mixing_base = mixingFrequency == 0 ? 0 : generalShardSize / mixingFrequency; + size_t dumping_base = weightDumpFrequency == 0 ? 0 : generalShardSize / weightDumpFrequency; + bool mix = evaluateModulo(shardPosition, mixing_base, actualBatchSize); + + // mix weights? + if (mix) { +#ifdef MPI_ENABLE + cerr << "Rank " << rank << ", epoch " << epoch << ", mixing weights.. " << endl; + // collect all weights in mixedWeights and divide by number of processes + mpi::reduce(world, mosesWeights, mixedWeights, SCCPlus(), 0); + + // mix confidence counts + //mpi::reduce(world, confidenceCounts, mixedConfidenceCounts, SCCPlus(), 0); + ScoreComponentCollection totalBinary; + if (sparseAverage) { + ScoreComponentCollection binary; + binary.SetToBinaryOf(mosesWeights); + mpi::reduce(world, binary, totalBinary, SCCPlus(), 0); + } + if (rank == 0) { + // divide by number of processes + if (sparseNoAverage) + mixedWeights.CoreDivideEquals(size); // average only core weights + else if (sparseAverage) + mixedWeights.DivideEquals(totalBinary); + else + mixedWeights.DivideEquals(size); + + // divide confidence counts + //mixedConfidenceCounts.DivideEquals(size); + + // normalise weights after averaging + if (normaliseWeights) { + mixedWeights.L1Normalise(); + } + + ++weightMixingThisEpoch; + + if (pruneZeroWeights) { + size_t pruned = mixedWeights.PruneZeroWeightFeatures(); + cerr << "Rank " << rank << ", epoch " << epoch << ", " + << pruned << " zero-weighted features pruned from mixedWeights." << endl; + + pruned = cumulativeWeights.PruneZeroWeightFeatures(); + cerr << "Rank " << rank << ", epoch " << epoch << ", " + << pruned << " zero-weighted features pruned from cumulativeWeights." << endl; + } + + if (featureCutoff != -1 && weightMixingThisEpoch == mixingFrequency) { + size_t pruned = mixedWeights.PruneSparseFeatures(featureCutoff); + cerr << "Rank " << rank << ", epoch " << epoch << ", " + << pruned << " features pruned from mixedWeights." << endl; + + pruned = cumulativeWeights.PruneSparseFeatures(featureCutoff); + cerr << "Rank " << rank << ", epoch " << epoch << ", " + << pruned << " features pruned from cumulativeWeights." << endl; + } + + if (weightMixingThisEpoch == mixingFrequency || reg_on_every_mix) { + if (l1_regularize) { + size_t pruned; + if (l1_reg_sparse) + pruned = mixedWeights.SparseL1Regularize(l1_lambda); + else + pruned = mixedWeights.L1Regularize(l1_lambda); + cerr << "Rank " << rank << ", epoch " << epoch << ", " + << "l1-reg. on mixedWeights with lambda=" << l1_lambda << ", pruned: " << pruned << endl; + } + if (l2_regularize) { + if (l2_reg_sparse) + mixedWeights.SparseL2Regularize(l2_lambda); + else + mixedWeights.L2Regularize(l2_lambda); + cerr << "Rank " << rank << ", epoch " << epoch << ", " + << "l2-reg. on mixedWeights with lambda=" << l2_lambda << endl; + } + } + } + + // broadcast average weights from process 0 + mpi::broadcast(world, mixedWeights, 0); + decoder->setWeights(mixedWeights); + mosesWeights = mixedWeights; + + // broadcast summed confidence counts + //mpi::broadcast(world, mixedConfidenceCounts, 0); + //confidenceCounts = mixedConfidenceCounts; +#endif +#ifndef MPI_ENABLE + //cerr << "\nRank " << rank << ", no mixing, weights: " << mosesWeights << endl; + mixedWeights = mosesWeights; +#endif + } // end mixing + + // Dump weights? + if (trainWithMultipleFolds || weightEpochDump == weightDumpFrequency) { + // dump mixed weights at end of every epoch to enable continuing a crashed experiment + // (for jackknife every time the weights are mixed) + ostringstream filename; + if (epoch < 10) + filename << weightDumpStem << "_mixed_0" << epoch; + else + filename << weightDumpStem << "_mixed_" << epoch; + + if (weightDumpFrequency > 1) + filename << "_" << weightEpochDump; + + mixedWeights.Save(filename.str()); + cerr << "Dumping mixed weights during epoch " << epoch << " to " << filename.str() << endl << endl; + } + if (dumpMixedWeights) { + if (mix && rank == 0 && !weightDumpStem.empty()) { + // dump mixed weights instead of average weights + ostringstream filename; + if (epoch < 10) + filename << weightDumpStem << "_0" << epoch; + else + filename << weightDumpStem << "_" << epoch; + + if (weightDumpFrequency > 1) + filename << "_" << weightEpochDump; + + cerr << "Dumping mixed weights during epoch " << epoch << " to " << filename.str() << endl << endl; + mixedWeights.Save(filename.str()); + ++weightEpochDump; + } + } + else { + if (evaluateModulo(shardPosition, dumping_base, actualBatchSize)) { + cerr << "Rank " << rank << ", epoch " << epoch << ", dump weights.. (pos: " << shardPosition << ", base: " << dumping_base << ")" << endl; + ScoreComponentCollection tmpAverageWeights(cumulativeWeights); + bool proceed = false; + if (accumulateWeights) { + if (numberOfUpdates > 0) { + tmpAverageWeights.DivideEquals(numberOfUpdates); + proceed = true; + } + } else { + if (numberOfUpdatesThisEpoch > 0) { + if (sparseNoAverage) // average only core weights + tmpAverageWeights.CoreDivideEquals(numberOfUpdatesThisEpoch); + else if (sparseAverage) + tmpAverageWeights.DivideEquals(cumulativeWeightsBinary); + else + tmpAverageWeights.DivideEquals(numberOfUpdatesThisEpoch); + proceed = true; + } + } + + if (proceed) { +#ifdef MPI_ENABLE + // average across processes + mpi::reduce(world, tmpAverageWeights, mixedAverageWeights, SCCPlus(), 0); + ScoreComponentCollection totalBinary; + if (sparseAverage) { + ScoreComponentCollection binary; + binary.SetToBinaryOf(mosesWeights); + mpi::reduce(world, binary, totalBinary, SCCPlus(), 0); + } +#endif +#ifndef MPI_ENABLE + mixedAverageWeights = tmpAverageWeights; +#endif + if (rank == 0 && !weightDumpStem.empty()) { + // divide by number of processes + if (sparseNoAverage) + mixedAverageWeights.CoreDivideEquals(size); // average only core weights + else if (sparseAverage) + mixedAverageWeights.DivideEquals(totalBinary); + else + mixedAverageWeights.DivideEquals(size); + + // normalise weights after averaging + if (normaliseWeights) { + mixedAverageWeights.L1Normalise(); + } + + // dump final average weights + ostringstream filename; + if (epoch < 10) { + filename << weightDumpStem << "_0" << epoch; + } else { + filename << weightDumpStem << "_" << epoch; + } + + if (weightDumpFrequency > 1) { + filename << "_" << weightEpochDump; + } + + /*if (accumulateWeights) { + cerr << "\nMixed average weights (cumulative) during epoch " << epoch << ": " << mixedAverageWeights << endl; + } else { + cerr << "\nMixed average weights during epoch " << epoch << ": " << mixedAverageWeights << endl; + }*/ + + cerr << "Dumping mixed average weights during epoch " << epoch << " to " << filename.str() << endl << endl; + mixedAverageWeights.Save(filename.str()); + ++weightEpochDump; + + if (weightEpochDump == weightDumpFrequency) { + if (l1_regularize) { + size_t pruned = mixedAverageWeights.SparseL1Regularize(l1_lambda); + cerr << "Rank " << rank << ", epoch " << epoch << ", " + << "l1-reg. on mixedAverageWeights with lambda=" << l1_lambda << ", pruned: " << pruned << endl; + + } + if (l2_regularize) { + mixedAverageWeights.SparseL2Regularize(l2_lambda); + cerr << "Rank " << rank << ", epoch " << epoch << ", " + << "l2-reg. on mixedAverageWeights with lambda=" << l2_lambda << endl; + } + + if (l1_regularize || l2_regularize) { + filename << "_reg"; + cerr << "Dumping regularized mixed average weights during epoch " << epoch << " to " << filename.str() << endl << endl; + mixedAverageWeights.Save(filename.str()); + } + } + + if (weightEpochDump == weightDumpFrequency && printFeatureCounts) { + // print out all features with counts + stringstream s1, s2; + s1 << "sparse_feature_hope_counts" << "_" << epoch; + s2 << "sparse_feature_fear_counts" << "_" << epoch; + ofstream sparseFeatureCountsHope(s1.str().c_str()); + ofstream sparseFeatureCountsFear(s2.str().c_str()); + + mixedAverageWeights.PrintSparseHopeFeatureCounts(sparseFeatureCountsHope); + mixedAverageWeights.PrintSparseFearFeatureCounts(sparseFeatureCountsFear); + sparseFeatureCountsHope.close(); + sparseFeatureCountsFear.close(); + } + } + } + }// end dumping + } // end if dump + } // end of shard loop, end of this epoch + cerr << "Rank " << rank << ", epoch " << epoch << ", end of epoch.." << endl; + + if (historyBleu || simpleHistoryBleu) { + cerr << "Bleu feature history after epoch " << epoch << endl; + decoder->printBleuFeatureHistory(cerr); + } + // cerr << "Rank " << rank << ", epoch " << epoch << ", sum of violated constraints: " << sumStillViolatedConstraints << endl; + + // Check whether there were any weight updates during this epoch + size_t sumUpdates; + size_t *sendbuf_uint, *recvbuf_uint; + sendbuf_uint = (size_t *) malloc(sizeof(size_t)); + recvbuf_uint = (size_t *) malloc(sizeof(size_t)); +#ifdef MPI_ENABLE + sendbuf_uint[0] = numberOfUpdatesThisEpoch; + recvbuf_uint[0] = 0; + MPI_Reduce(sendbuf_uint, recvbuf_uint, 1, MPI_UNSIGNED, MPI_SUM, 0, world); + sumUpdates = recvbuf_uint[0]; +#endif +#ifndef MPI_ENABLE + sumUpdates = numberOfUpdatesThisEpoch; +#endif + if (rank == 0 && sumUpdates == 0) { + cerr << "\nNo weight updates during this epoch.. stopping." << endl; + stop = true; +#ifdef MPI_ENABLE + mpi::broadcast(world, stop, 0); +#endif + } + + if (!stop) { + // Test if weights have converged + if (weightConvergence) { + bool reached = true; + if (rank == 0 && (epoch >= 2)) { + ScoreComponentCollection firstDiff, secondDiff; + if (dumpMixedWeights) { + firstDiff = mixedWeights; + firstDiff.MinusEquals(mixedWeightsPrevious); + secondDiff = mixedWeights; + secondDiff.MinusEquals(mixedWeightsBeforePrevious); + } + else { + firstDiff = mixedAverageWeights; + firstDiff.MinusEquals(mixedAverageWeightsPrevious); + secondDiff = mixedAverageWeights; + secondDiff.MinusEquals(mixedAverageWeightsBeforePrevious); + } + VERBOSE(1, "Average weight changes since previous epoch: " << firstDiff << " (max: " << firstDiff.GetLInfNorm() << ")" << endl); + VERBOSE(1, "Average weight changes since before previous epoch: " << secondDiff << " (max: " << secondDiff.GetLInfNorm() << ")" << endl << endl); + + // check whether stopping criterion has been reached + // (both difference vectors must have all weight changes smaller than min_weight_change) + if (firstDiff.GetLInfNorm() >= min_weight_change) + reached = false; + if (secondDiff.GetLInfNorm() >= min_weight_change) + reached = false; + if (reached) { + // stop MIRA + stop = true; + cerr << "\nWeights have converged after epoch " << epoch << ".. stopping MIRA." << endl; + ScoreComponentCollection dummy; + ostringstream endfilename; + endfilename << "stopping"; + dummy.Save(endfilename.str()); + } + } + + mixedWeightsBeforePrevious = mixedWeightsPrevious; + mixedWeightsPrevious = mixedWeights; + mixedAverageWeightsBeforePrevious = mixedAverageWeightsPrevious; + mixedAverageWeightsPrevious = mixedAverageWeights; +#ifdef MPI_ENABLE + mpi::broadcast(world, stop, 0); +#endif + } //end if (weightConvergence) + } + } // end of epoch loop + +#ifdef MPI_ENABLE + MPI_Finalize(); +#endif + + time(&now); + cerr << "Rank " << rank << ", " << ctime(&now); + + if (rank == 0) { + ScoreComponentCollection dummy; + ostringstream endfilename; + endfilename << "finished"; + dummy.Save(endfilename.str()); + } + + delete decoder; + exit(0); +} + +bool loadSentences(const string& filename, vector<string>& sentences) { + ifstream in(filename.c_str()); + if (!in) + return false; + string line; + while (getline(in, line)) + sentences.push_back(line); + return true; +} + +bool evaluateModulo(size_t shard_position, size_t mix_or_dump_base, size_t actual_batch_size) { + if (mix_or_dump_base == 0) return 0; + if (actual_batch_size > 1) { + bool mix_or_dump = false; + size_t numberSubtracts = actual_batch_size; + do { + if (shard_position % mix_or_dump_base == 0) { + mix_or_dump = true; + break; + } + --shard_position; + --numberSubtracts; + } while (numberSubtracts > 0); + return mix_or_dump; + } + else { + return ((shard_position % mix_or_dump_base) == 0); + } +} + +void printFeatureValues(vector<vector<ScoreComponentCollection> > &featureValues) { + for (size_t i = 0; i < featureValues.size(); ++i) { + for (size_t j = 0; j < featureValues[i].size(); ++j) { + cerr << featureValues[i][j] << endl; + } + } + cerr << endl; +} + +void deleteTranslations(vector<vector<const Word*> > &translations) { + for (size_t i = 0; i < translations.size(); ++i) { + for (size_t j = 0; j < translations[i].size(); ++j) { + delete translations[i][j]; + } + } +} + +void decodeHopeOrFear(size_t rank, size_t size, size_t decode, string filename, vector<string> &inputSentences, MosesDecoder* decoder, size_t n, float bleuWeight) { + if (decode == 1) + cerr << "Rank " << rank << ", decoding dev input set according to hope objective.. " << endl; + else if (decode == 2) + cerr << "Rank " << rank << ", decoding dev input set according to fear objective.. " << endl; + else + cerr << "Rank " << rank << ", decoding dev input set according to normal objective.. " << endl; + + // Create shards according to the number of processes used + vector<size_t> order; + for (size_t i = 0; i < inputSentences.size(); ++i) + order.push_back(i); + + vector<size_t> shard; + float shardSize = (float) (order.size()) / size; + size_t shardStart = (size_t) (shardSize * rank); + size_t shardEnd = (size_t) (shardSize * (rank + 1)); + if (rank == size - 1) { + shardEnd = inputSentences.size(); + shardSize = shardEnd - shardStart; + } + VERBOSE(1, "Rank " << rank << ", shard start: " << shardStart << " Shard end: " << shardEnd << endl); + VERBOSE(1, "Rank " << rank << ", shard size: " << shardSize << endl); + shard.resize(shardSize); + copy(order.begin() + shardStart, order.begin() + shardEnd, shard.begin()); + + // open files for writing + stringstream fname; + fname << filename << ".rank" << rank; + filename = fname.str(); + ostringstream filename_nbest; + filename_nbest << filename << "." << n << "best"; + ofstream out(filename.c_str()); + ofstream nbest_out((filename_nbest.str()).c_str()); + if (!out) { + ostringstream msg; + msg << "Unable to open " << fname.str(); + throw runtime_error(msg.str()); + } + if (!nbest_out) { + ostringstream msg; + msg << "Unable to open " << filename_nbest; + throw runtime_error(msg.str()); + } + + for (size_t i = 0; i < shard.size(); ++i) { + size_t sid = shard[i]; + string& input = inputSentences[sid]; + + vector<vector<ScoreComponentCollection> > dummyFeatureValues; + vector<vector<float> > dummyBleuScores; + vector<vector<float> > dummyModelScores; + + vector<ScoreComponentCollection> newFeatureValues; + vector<float> newScores; + dummyFeatureValues.push_back(newFeatureValues); + dummyBleuScores.push_back(newScores); + dummyModelScores.push_back(newScores); + + float factor = 0.0; + if (decode == 1) factor = 1.0; + if (decode == 2) factor = -1.0; + cerr << "Rank " << rank << ", translating sentence " << sid << endl; + bool realBleu = false; + vector< vector<const Word*> > nbestOutput = decoder->getNBest(input, sid, n, factor, bleuWeight, dummyFeatureValues[0], + dummyBleuScores[0], dummyModelScores[0], n, realBleu, true, false, rank, 0, ""); + cerr << endl; + decoder->cleanup(StaticData::Instance().GetSearchAlgorithm() == ChartDecoding); + + for (size_t i = 0; i < nbestOutput.size(); ++i) { + vector<const Word*> output = nbestOutput[i]; + stringstream translation; + for (size_t k = 0; k < output.size(); ++k) { + Word* w = const_cast<Word*>(output[k]); + translation << w->GetString(0); + translation << " "; + } + + if (i == 0) + out << translation.str() << endl; + nbest_out << sid << " ||| " << translation.str() << " ||| " << dummyFeatureValues[0][i] << + " ||| " << dummyModelScores[0][i] << " ||| sBleu=" << dummyBleuScores[0][i] << endl; + } + } + + out.close(); + nbest_out.close(); + cerr << "Closing files " << filename << " and " << filename_nbest.str() << endl; + +#ifdef MPI_ENABLE + MPI_Finalize(); +#endif + + time_t now; + time(&now); + cerr << "Rank " << rank << ", " << ctime(&now); + + delete decoder; + exit(0); +} + +void applyLearningRates(vector<vector<ScoreComponentCollection> > &featureValues, float core_r0, float sparse_r0) { + for (size_t i=0; i<featureValues.size(); ++i) // each item in batch + for (size_t j=0; j<featureValues[i].size(); ++j) // each item in nbest + featureValues[i][j].MultiplyEquals(core_r0, sparse_r0); +} + +void applyPerFeatureLearningRates(vector<vector<ScoreComponentCollection> > &featureValues, ScoreComponentCollection featureLearningRates, float sparse_r0) { + for (size_t i=0; i<featureValues.size(); ++i) // each item in batch + for (size_t j=0; j<featureValues[i].size(); ++j) // each item in nbest + featureValues[i][j].MultiplyEqualsBackoff(featureLearningRates, sparse_r0); +} + +void scaleFeatureScore(ScoreProducer *sp, float scaling_factor, vector<vector<ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch) { + string name = sp->GetScoreProducerWeightShortName(); + + // scale down score + float featureScore; + for (size_t i=0; i<featureValues.size(); ++i) { // each item in batch + for (size_t j=0; j<featureValues[i].size(); ++j) { // each item in nbest + featureScore = featureValues[i][j].GetScoreForProducer(sp); + featureValues[i][j].Assign(sp, featureScore*scaling_factor); + //cerr << "Rank " << rank << ", epoch " << epoch << ", " << name << " score scaled from " << featureScore << " to " << featureScore/scaling_factor << endl; + } + } +} + +void scaleFeatureScores(ScoreProducer *sp, float scaling_factor, vector<vector<ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch) { + string name = sp->GetScoreProducerWeightShortName(); + + // scale down score + for (size_t i=0; i<featureValues.size(); ++i) { // each item in batch + for (size_t j=0; j<featureValues[i].size(); ++j) { // each item in nbest + vector<float> featureScores = featureValues[i][j].GetScoresForProducer(sp); + for (size_t k=0; k<featureScores.size(); ++k) + featureScores[k] *= scaling_factor; + featureValues[i][j].Assign(sp, featureScores); + //cerr << "Rank " << rank << ", epoch " << epoch << ", " << name << " score scaled from " << featureScore << " to " << featureScore/scaling_factor << endl; + } + } +} diff --git a/mira/Main.h b/mira/Main.h new file mode 100644 index 000000000..23db36c36 --- /dev/null +++ b/mira/Main.h @@ -0,0 +1,58 @@ +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2010 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ +#ifndef MAIN_H_ +#define MAIN_H_ + +#include <vector> + +#include "ScoreComponentCollection.h" +#include "Word.h" +#include "ScoreProducer.h" +#include "Decoder.h" + +typedef std::map<const Moses::ScoreProducer*, std::vector< float > > ProducerWeightMap; +typedef std::pair<const Moses::ScoreProducer*, std::vector< float > > ProducerWeightPair; + +template <class T> bool from_string(T& t, const std::string& s, std::ios_base& (*f)(std::ios_base&)) +{ + std::istringstream iss(s); + return !(iss >> f >> t).fail(); +} + +struct RandomIndex { + ptrdiff_t operator()(ptrdiff_t max) { + srand(time(0)); // Initialize random number generator with current time. + return static_cast<ptrdiff_t> (rand() % max); + } +}; + +//void OutputNBestList(const MosesChart::TrellisPathList &nBestList, const TranslationSystem* system, long translationId); +bool loadSentences(const std::string& filename, std::vector<std::string>& sentences); +bool evaluateModulo(size_t shard_position, size_t mix_or_dump_base, size_t actual_batch_size); +void printFeatureValues(std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues); +void ignoreCoreFeatures(std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, ProducerWeightMap &coreWeightMap); +void takeLogs(std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, size_t base); +void deleteTranslations(std::vector<std::vector<const Moses::Word*> > &translations); +void decodeHopeOrFear(size_t rank, size_t size, size_t decode, std::string decode_filename, std::vector<std::string> &inputSentences, Mira::MosesDecoder* decoder, size_t n, float bleuWeight); +void applyLearningRates(std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, float core_r0, float sparse_r0); +void applyPerFeatureLearningRates(std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, Moses::ScoreComponentCollection featureLearningRates, float sparse_r0); +void scaleFeatureScore(Moses::ScoreProducer *sp, float scaling_factor, std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch); +void scaleFeatureScores(Moses::ScoreProducer *sp, float scaling_factor, std::vector<std::vector<Moses::ScoreComponentCollection> > &featureValues, size_t rank, size_t epoch); + +#endif /* MAIN_H_ */ diff --git a/mira/Makefile.am b/mira/Makefile.am new file mode 100644 index 000000000..cd490c853 --- /dev/null +++ b/mira/Makefile.am @@ -0,0 +1,14 @@ +bin_PROGRAMS = mira +mira_SOURCES = \ + Decoder.cpp \ + Main.cpp \ + MiraOptimiser.cpp \ + Perceptron.cpp \ + Hildreth.cpp + + +AM_CPPFLAGS = -W -Wall -Wno-unused -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES -I$(top_srcdir)/moses/src -I$(top_srcdir)/moses-chart/src $(BOOST_CPPFLAGS) + + +mira_LDADD = $(top_builddir)/moses/src/libmoses.la -L$(top_srcdir)/moses/src -lmoses -L$(top_srcdir)/OnDiskPt/src -lOnDiskPt $(top_srcdir)/util/libkenutil.la $(top_srcdir)/lm/libkenlm.la $(BOOST_LDFLAGS) $(BOOST_SERIALIZATION_LIBS) $(BOOST_PROGRAM_OPTIONS_LDFLAGS) $(BOOST_PROGRAM_OPTIONS_LIBS) $(BOOST_MPI_LIBS) +mira_DEPENDENCIES = $(top_srcdir)/moses/src/libmoses.la diff --git a/mira/MiraOptimiser.cpp b/mira/MiraOptimiser.cpp new file mode 100644 index 000000000..d77cef759 --- /dev/null +++ b/mira/MiraOptimiser.cpp @@ -0,0 +1,749 @@ +#include "Optimiser.h" +#include "Hildreth.h" +#include "StaticData.h" + +using namespace Moses; +using namespace std; + +namespace Mira { + +size_t MiraOptimiser::updateWeights( + ScoreComponentCollection& weightUpdate, + const vector<vector<ScoreComponentCollection> >& featureValues, + const vector<vector<float> >& losses, + const vector<vector<float> >& bleuScores, + const vector<vector<float> >& modelScores, + const vector<ScoreComponentCollection>& oracleFeatureValues, + const vector<float> oracleBleuScores, + const vector<float> oracleModelScores, + float learning_rate, + size_t rank, + size_t epoch) { + + // vector of feature values differences for all created constraints + vector<ScoreComponentCollection> featureValueDiffs; + vector<float> lossMinusModelScoreDiffs; + vector<float> all_losses; + + // most violated constraint in batch + ScoreComponentCollection max_batch_featureValueDiff; + + // Make constraints for new hypothesis translations + float epsilon = 0.0001; + int violatedConstraintsBefore = 0; + float oldDistanceFromOptimum = 0; + // iterate over input sentences (1 (online) or more (batch)) + for (size_t i = 0; i < featureValues.size(); ++i) { + //size_t sentenceId = sentenceIds[i]; + // iterate over hypothesis translations for one input sentence + for (size_t j = 0; j < featureValues[i].size(); ++j) { + ScoreComponentCollection featureValueDiff = oracleFeatureValues[i]; + featureValueDiff.MinusEquals(featureValues[i][j]); + + // cerr << "Rank " << rank << ", epoch " << epoch << ", feature value diff: " << featureValueDiff << endl; + if (featureValueDiff.GetL1Norm() == 0) { + cerr << "Rank " << rank << ", epoch " << epoch << ", features equal --> skip" << endl; + continue; + } + + float loss = losses[i][j]; + + // check if constraint is violated + bool violated = false; +// float modelScoreDiff = featureValueDiff.InnerProduct(currWeights); + float modelScoreDiff = oracleModelScores[i] - modelScores[i][j]; + float diff = 0; + + if (loss > modelScoreDiff) + diff = loss - modelScoreDiff; + cerr << "Rank " << rank << ", epoch " << epoch << ", constraint: " << modelScoreDiff << " >= " << loss << " (current violation: " << diff << ")" << endl; + if (diff > epsilon) + violated = true; + + if (m_normaliseMargin) { + modelScoreDiff = (2*m_sigmoidParam/(1 + exp(-modelScoreDiff))) - m_sigmoidParam; + loss = (2*m_sigmoidParam/(1 + exp(-loss))) - m_sigmoidParam; + diff = 0; + if (loss > modelScoreDiff) { + diff = loss - modelScoreDiff; + } + cerr << "Rank " << rank << ", epoch " << epoch << ", normalised constraint: " << modelScoreDiff << " >= " << loss << " (current violation: " << diff << ")" << endl; + } + + if (m_scale_margin) { + diff *= oracleBleuScores[i]; + cerr << "Rank " << rank << ", epoch " << epoch << ", scaling margin with oracle bleu score " << oracleBleuScores[i] << endl; + } + + featureValueDiffs.push_back(featureValueDiff); + lossMinusModelScoreDiffs.push_back(diff); + all_losses.push_back(loss); + if (violated) { + ++violatedConstraintsBefore; + oldDistanceFromOptimum += diff; + } + } + } + + // run optimisation: compute alphas for all given constraints + vector<float> alphas; + ScoreComponentCollection summedUpdate; + if (violatedConstraintsBefore > 0) { + cerr << "Rank " << rank << ", epoch " << epoch << ", number of constraints passed to optimizer: " << + featureValueDiffs.size() << " (of which violated: " << violatedConstraintsBefore << ")" << endl; + if (m_slack != 0) { + alphas = Hildreth::optimise(featureValueDiffs, lossMinusModelScoreDiffs, m_slack); + } else { + alphas = Hildreth::optimise(featureValueDiffs, lossMinusModelScoreDiffs); + } + + // Update the weight vector according to the alphas and the feature value differences + // * w' = w' + SUM alpha_i * (h_i(oracle) - h_i(hypothesis)) + for (size_t k = 0; k < featureValueDiffs.size(); ++k) { + float alpha = alphas[k]; + cerr << "Rank " << rank << ", epoch " << epoch << ", alpha: " << alpha << endl; + ScoreComponentCollection update(featureValueDiffs[k]); + update.MultiplyEquals(alpha); + + // sum updates + summedUpdate.PlusEquals(update); + } + } + else { + cerr << "Rank " << rank << ", epoch " << epoch << ", no constraint violated for this batch" << endl; +// return 0; + return 1; + } + + // apply learning rate + if (learning_rate != 1) { + cerr << "Rank " << rank << ", epoch " << epoch << ", apply learning rate " << learning_rate << " to update." << endl; + summedUpdate.MultiplyEquals(learning_rate); + } + + // scale update by BLEU of oracle (for batch size 1 only) + if (oracleBleuScores.size() == 1) { + if (m_scale_update) { + cerr << "Rank " << rank << ", epoch " << epoch << ", scaling summed update with oracle bleu score " << oracleBleuScores[0] << endl; + summedUpdate.MultiplyEquals(oracleBleuScores[0]); + } + } + + // cerr << "Rank " << rank << ", epoch " << epoch << ", update: " << summedUpdate << endl; + weightUpdate.PlusEquals(summedUpdate); + + // Sanity check: are there still violated constraints after optimisation? +/* int violatedConstraintsAfter = 0; + float newDistanceFromOptimum = 0; + for (size_t i = 0; i < featureValueDiffs.size(); ++i) { + float modelScoreDiff = featureValueDiffs[i].InnerProduct(currWeights); + float loss = all_losses[i]; + float diff = loss - modelScoreDiff; + if (diff > epsilon) { + ++violatedConstraintsAfter; + newDistanceFromOptimum += diff; + } + } + VERBOSE(1, "Rank " << rank << ", epoch " << epoch << ", violated constraint before: " << violatedConstraintsBefore << ", after: " << violatedConstraintsAfter << ", change: " << violatedConstraintsBefore - violatedConstraintsAfter << endl); + VERBOSE(1, "Rank " << rank << ", epoch " << epoch << ", error before: " << oldDistanceFromOptimum << ", after: " << newDistanceFromOptimum << ", change: " << oldDistanceFromOptimum - newDistanceFromOptimum << endl);*/ +// return violatedConstraintsAfter; + return 0; +} + +size_t MiraOptimiser::updateWeightsHopeFear( + Moses::ScoreComponentCollection& weightUpdate, + const std::vector< std::vector<Moses::ScoreComponentCollection> >& featureValuesHope, + const std::vector< std::vector<Moses::ScoreComponentCollection> >& featureValuesFear, + const std::vector<std::vector<float> >& bleuScoresHope, + const std::vector<std::vector<float> >& bleuScoresFear, + const std::vector<std::vector<float> >& modelScoresHope, + const std::vector<std::vector<float> >& modelScoresFear, + float learning_rate, + size_t rank, + size_t epoch, + int updatePosition) { + + // vector of feature values differences for all created constraints + vector<ScoreComponentCollection> featureValueDiffs; + vector<float> lossMinusModelScoreDiffs; + vector<float> modelScoreDiffs; + vector<float> all_losses; + + // most violated constraint in batch + ScoreComponentCollection max_batch_featureValueDiff; + + // Make constraints for new hypothesis translations + float epsilon = 0.0001; + int violatedConstraintsBefore = 0; + float oldDistanceFromOptimum = 0; + + // iterate over input sentences (1 (online) or more (batch)) + for (size_t i = 0; i < featureValuesHope.size(); ++i) { + if (updatePosition != -1) { + if (i < updatePosition) + continue; + else if (i > updatePosition) + break; + } + + // Pick all pairs[j,j] of hope and fear translations for one input sentence + for (size_t j = 0; j < featureValuesHope[i].size(); ++j) { + ScoreComponentCollection featureValueDiff = featureValuesHope[i][j]; + featureValueDiff.MinusEquals(featureValuesFear[i][j]); + //cerr << "Rank " << rank << ", epoch " << epoch << ", feature value diff: " << featureValueDiff << endl; + if (featureValueDiff.GetL1Norm() == 0) { + cerr << "Rank " << rank << ", epoch " << epoch << ", features equal --> skip" << endl; + continue; + } + + float loss = bleuScoresHope[i][j] - bleuScoresFear[i][j]; + + // check if constraint is violated + bool violated = false; + //float modelScoreDiff = featureValueDiff.InnerProduct(currWeights); + float modelScoreDiff = modelScoresHope[i][j] - modelScoresFear[i][j]; + float diff = 0; + if (loss > modelScoreDiff) + diff = loss - modelScoreDiff; + cerr << "Rank " << rank << ", epoch " << epoch << ", constraint: " << modelScoreDiff << " >= " << loss << " (current violation: " << diff << ")" << endl; + + if (diff > epsilon) + violated = true; + + if (m_normaliseMargin) { + modelScoreDiff = (2*m_sigmoidParam/(1 + exp(-modelScoreDiff))) - m_sigmoidParam; + loss = (2*m_sigmoidParam/(1 + exp(-loss))) - m_sigmoidParam; + diff = 0; + if (loss > modelScoreDiff) { + diff = loss - modelScoreDiff; + } + cerr << "Rank " << rank << ", epoch " << epoch << ", normalised constraint: " << modelScoreDiff << " >= " << loss << " (current violation: " << diff << ")" << endl; + } + + if (m_scale_margin) { + diff *= bleuScoresHope[i][j]; + cerr << "Rank " << rank << ", epoch " << epoch << ", scaling margin with oracle bleu score " << bleuScoresHope[i][j] << endl; + } + + featureValueDiffs.push_back(featureValueDiff); + lossMinusModelScoreDiffs.push_back(diff); + modelScoreDiffs.push_back(modelScoreDiff); + all_losses.push_back(loss); + if (violated) { + ++violatedConstraintsBefore; + oldDistanceFromOptimum += diff; + } + } + } + + // run optimisation: compute alphas for all given constraints + vector<float> alphas; + ScoreComponentCollection summedUpdate; + if (violatedConstraintsBefore > 0) { + cerr << "Rank " << rank << ", epoch " << epoch << ", number of constraints passed to optimizer: " << + featureValueDiffs.size() << " (of which violated: " << violatedConstraintsBefore << ")" << endl; + if (m_slack != 0) { + alphas = Hildreth::optimise(featureValueDiffs, lossMinusModelScoreDiffs, m_slack); + } else { + alphas = Hildreth::optimise(featureValueDiffs, lossMinusModelScoreDiffs); + } + + // Update the weight vector according to the alphas and the feature value differences + // * w' = w' + SUM alpha_i * (h_i(oracle) - h_i(hypothesis)) + for (size_t k = 0; k < featureValueDiffs.size(); ++k) { + float alpha = alphas[k]; + cerr << "Rank " << rank << ", epoch " << epoch << ", alpha: " << alpha << endl; + if (alpha != 0) { + // apply boosting factor + if (m_boost && modelScoreDiffs[k] <= 0) { + // factor between 1.5 and 3 (for Bleu scores between 5 and 20, the factor is within the boundaries) + float factor = min(1.5, log2(bleuScoresHope[0][0])); // TODO: make independent of number of oracles!! + factor = min(3.0f, factor); + alpha = alpha * factor; + cerr << "Rank " << rank << ", epoch " << epoch << ", apply boosting factor " << factor << " to update." << endl; + } + + ScoreComponentCollection update(featureValueDiffs[k]); + update.MultiplyEquals(alpha); + + // sum updates + summedUpdate.PlusEquals(update); + } + } + } + else { + cerr << "Rank " << rank << ", epoch " << epoch << ", no constraint violated for this batch" << endl; + // return 0; + return 1; + } + + // apply learning rate + if (learning_rate != 1) { + cerr << "Rank " << rank << ", epoch " << epoch << ", apply learning rate " << learning_rate << " to update." << endl; + summedUpdate.MultiplyEquals(learning_rate); + } + + // scale update by BLEU of oracle (for batch size 1 only) + if (featureValuesHope.size() == 1) { + if (m_scale_update) { + cerr << "Rank " << rank << ", epoch " << epoch << ", scaling summed update with oracle bleu score " << bleuScoresHope[0][0] << endl; + summedUpdate.MultiplyEquals(bleuScoresHope[0][0]); + } + } + + //cerr << "Rank " << rank << ", epoch " << epoch << ", update: " << summedUpdate << endl; + weightUpdate.PlusEquals(summedUpdate); + + // Sanity check: are there still violated constraints after optimisation? +/* int violatedConstraintsAfter = 0; + float newDistanceFromOptimum = 0; + for (size_t i = 0; i < featureValueDiffs.size(); ++i) { + float modelScoreDiff = featureValueDiffs[i].InnerProduct(currWeights); + float loss = all_losses[i]; + float diff = loss - modelScoreDiff; + if (diff > epsilon) { + ++violatedConstraintsAfter; + newDistanceFromOptimum += diff; + } + } + VERBOSE(1, "Rank " << rank << ", epoch " << epoch << ", violated constraint before: " << violatedConstraintsBefore << ", after: " << violatedConstraintsAfter << ", change: " << violatedConstraintsBefore - violatedConstraintsAfter << endl); + VERBOSE(1, "Rank " << rank << ", epoch " << epoch << ", error before: " << oldDistanceFromOptimum << ", after: " << newDistanceFromOptimum << ", change: " << oldDistanceFromOptimum - newDistanceFromOptimum << endl);*/ +// return violatedConstraintsAfter; + return 0; +} + +size_t MiraOptimiser::updateWeightsAnalytically( + ScoreComponentCollection& weightUpdate, + ScoreComponentCollection& featureValuesHope, + ScoreComponentCollection& featureValuesFear, + float bleuScoreHope, + float bleuScoreFear, + float modelScoreHope, + float modelScoreFear, + float learning_rate, + size_t rank, + size_t epoch) { + + float epsilon = 0.0001; + float oldDistanceFromOptimum = 0; + bool constraintViolatedBefore = false; + + // cerr << "Rank " << rank << ", epoch " << epoch << ", hope: " << featureValuesHope << endl; + // cerr << "Rank " << rank << ", epoch " << epoch << ", fear: " << featureValuesFear << endl; + ScoreComponentCollection featureValueDiff = featureValuesHope; + featureValueDiff.MinusEquals(featureValuesFear); + if (featureValueDiff.GetL1Norm() == 0) { + cerr << "Rank " << rank << ", epoch " << epoch << ", features equal --> skip" << endl; + return 1; + } + +// cerr << "Rank " << rank << ", epoch " << epoch << ", hope - fear: " << featureValueDiff << endl; +// float modelScoreDiff = featureValueDiff.InnerProduct(currWeights); + float modelScoreDiff = modelScoreHope - modelScoreFear; + float loss = bleuScoreHope - bleuScoreFear; + float diff = 0; + if (loss > modelScoreDiff) + diff = loss - modelScoreDiff; + cerr << "Rank " << rank << ", epoch " << epoch << ", constraint: " << modelScoreDiff << " >= " << loss << " (current violation: " << diff << ")" << endl; + + if (m_normaliseMargin) { + modelScoreDiff = (2*m_sigmoidParam/(1 + exp(-modelScoreDiff))) - m_sigmoidParam; + loss = (2*m_sigmoidParam/(1 + exp(-loss))) - m_sigmoidParam; + if (loss > modelScoreDiff) + diff = loss - modelScoreDiff; + cerr << "Rank " << rank << ", epoch " << epoch << ", normalised constraint: " << modelScoreDiff << " >= " << loss << " (current violation: " << diff << ")" << endl; + } + + if (m_scale_margin) { + diff *= bleuScoreHope; + cerr << "Rank " << rank << ", epoch " << epoch << ", scaling margin with oracle bleu score " << bleuScoreHope << endl; + } + if (m_scale_margin_precision) { + diff *= (1+m_precision); + cerr << "Rank " << rank << ", epoch " << epoch << ", scaling margin with 1+precision: " << (1+m_precision) << endl; + } + + if (diff > epsilon) { + // squash it between 0 and 1 + //diff = tanh(diff); + //diff = (2/(1 + pow(2,-diff))) - 1; + /* if (m_normaliseMargin) { + diff = (2/(1 + exp(-diff))) - 1; + cerr << "Rank " << rank << ", epoch " << epoch << ", new margin: " << diff << endl; + }*/ + + // constraint violated + oldDistanceFromOptimum += diff; + constraintViolatedBefore = true; + + // compute alpha for given constraint: (loss - model score diff) / || feature value diff ||^2 + // featureValueDiff.GetL2Norm() * featureValueDiff.GetL2Norm() == featureValueDiff.InnerProduct(featureValueDiff) + // from Crammer&Singer 2006: alpha = min {C , l_t/ ||x||^2} + float squaredNorm = featureValueDiff.GetL2Norm() * featureValueDiff.GetL2Norm(); + + float alpha = diff / squaredNorm; + cerr << "Rank " << rank << ", epoch " << epoch << ", unclipped alpha: " << alpha << endl; + if (m_slack > 0 ) { + if (alpha > m_slack) { + alpha = m_slack; + } + else if (alpha < m_slack*(-1)) { + alpha = m_slack*(-1); + } + } + + // apply learning rate + if (learning_rate != 1) + alpha = alpha * learning_rate; + + if (m_scale_update) { + cerr << "Rank " << rank << ", epoch " << epoch << ", scaling update with oracle bleu score " << bleuScoreHope << endl; + alpha *= bleuScoreHope; + } + if (m_scale_update_precision) { + cerr << "Rank " << rank << ", epoch " << epoch << ", scaling update with 1+precision: " << (1+m_precision) << endl; + alpha *= (1+m_precision); + } + + cerr << "Rank " << rank << ", epoch " << epoch << ", clipped/scaled alpha: " << alpha << endl; + + // apply boosting factor + if (m_boost && modelScoreDiff <= 0) { + // factor between 1.5 and 3 (for Bleu scores between 5 and 20, the factor is within the boundaries) + float factor = min(1.5, log2(bleuScoreHope)); + factor = min(3.0f, factor); + alpha = alpha * factor; + cerr << "Rank " << rank << ", epoch " << epoch << ", boosted alpha: " << alpha << endl; + } + + featureValueDiff.MultiplyEquals(alpha); + weightUpdate.PlusEquals(featureValueDiff); +// cerr << "Rank " << rank << ", epoch " << epoch << ", update: " << weightUpdate << endl; + } + + if (!constraintViolatedBefore) { + // constraint satisfied, nothing to do + cerr << "Rank " << rank << ", epoch " << epoch << ", constraint already satisfied" << endl; + return 1; + } + + // sanity check: constraint still violated after optimisation? +/* ScoreComponentCollection newWeights(currWeights); + newWeights.PlusEquals(weightUpdate); + bool constraintViolatedAfter = false; + float newDistanceFromOptimum = 0; + featureValueDiff = featureValuesHope; + featureValueDiff.MinusEquals(featureValuesFear); + modelScoreDiff = featureValueDiff.InnerProduct(newWeights); + diff = loss - modelScoreDiff; + // approximate comparison between floats! + if (diff > epsilon) { + constraintViolatedAfter = true; + newDistanceFromOptimum += (loss - modelScoreDiff); + } + + float hopeScore = featureValuesHope.InnerProduct(newWeights); + float fearScore = featureValuesFear.InnerProduct(newWeights); + cerr << "New hope score: " << hopeScore << endl; + cerr << "New fear score: " << fearScore << endl; + + VERBOSE(0, "Rank " << rank << ", epoch " << epoch << ", check, constraint violated before? " << constraintViolatedBefore << ", after? " << constraintViolatedAfter << endl); + VERBOSE(0, "Rank " << rank << ", epoch " << epoch << ", check, error before: " << oldDistanceFromOptimum << ", after: " << newDistanceFromOptimum << ", change: " << oldDistanceFromOptimum - newDistanceFromOptimum << endl); +*/ + return 0; +} + +size_t MiraOptimiser::updateWeightsHopeFearSelective( + Moses::ScoreComponentCollection& weightUpdate, + const std::vector< std::vector<Moses::ScoreComponentCollection> >& featureValuesHope, + const std::vector< std::vector<Moses::ScoreComponentCollection> >& featureValuesFear, + const std::vector<std::vector<float> >& bleuScoresHope, + const std::vector<std::vector<float> >& bleuScoresFear, + const std::vector<std::vector<float> >& modelScoresHope, + const std::vector<std::vector<float> >& modelScoresFear, + float learning_rate, + size_t rank, + size_t epoch, + int updatePosition) { + + // vector of feature values differences for all created constraints + vector<ScoreComponentCollection> nonZeroFeatures; + vector<float> lossMinusModelScoreDiffs; + + // Make constraints for new hypothesis translations + float epsilon = 0.0001; + int violatedConstraintsBefore = 0; + + // iterate over input sentences (1 (online) or more (batch)) + for (size_t i = 0; i < featureValuesHope.size(); ++i) { + if (updatePosition != -1) { + if (i < updatePosition) + continue; + else if (i > updatePosition) + break; + } + + // Pick all pairs[j,j] of hope and fear translations for one input sentence + for (size_t j = 0; j < featureValuesHope[i].size(); ++j) { + ScoreComponentCollection featureValueDiff = featureValuesHope[i][j]; + featureValueDiff.MinusEquals(featureValuesFear[i][j]); + if (featureValueDiff.GetL1Norm() == 0) { + cerr << "Rank " << rank << ", epoch " << epoch << ", features equal --> skip" << endl; + continue; + } + + // check if constraint is violated + float loss = bleuScoresHope[i][j] - bleuScoresFear[i][j]; + float modelScoreDiff = modelScoresHope[i][j] - modelScoresFear[i][j]; + float diff = 0; + if (loss > modelScoreDiff) + diff = loss - modelScoreDiff; + if (diff > epsilon) + ++violatedConstraintsBefore; + cerr << "Rank " << rank << ", epoch " << epoch << ", constraint: " << modelScoreDiff << " >= " << loss << " (current violation: " << diff << ")" << endl; + + // iterate over difference vector and add a constraint for every non-zero feature + FVector features = featureValueDiff.GetScoresVector(); + size_t n_core = 0, n_sparse = 0, n_sparse_hope = 0, n_sparse_fear = 0; + for (size_t i=0; i<features.coreSize(); ++i) { + if (features[i] != 0.0) { + ++n_core; + ScoreComponentCollection f; + f.Assign(i, features[i]); + nonZeroFeatures.push_back(f); + } + } + + vector<ScoreComponentCollection> nonZeroFeaturesHope; + vector<ScoreComponentCollection> nonZeroFeaturesFear; + for (FVector::iterator i = features.begin(); i != features.end(); ++i) { + if (i->second != 0.0) { + ScoreComponentCollection f; + f.Assign((i->first).name(), i->second); + cerr << "Rank " << rank << ", epoch " << epoch << ", f: " << f << endl; + + if (i->second > 0.0) { + ++n_sparse_hope; + nonZeroFeaturesHope.push_back(f); + } + else { + ++n_sparse_fear; + nonZeroFeaturesFear.push_back(f); + } + } + } + + float n = n_core + n_sparse_hope + n_sparse_fear; + for (size_t i=0; i<n_core; ++i) + lossMinusModelScoreDiffs.push_back(diff/n); + for (size_t i=0; i<n_sparse_hope; ++i) { + nonZeroFeatures.push_back(nonZeroFeaturesHope[i]); + lossMinusModelScoreDiffs.push_back((diff/n)*1.1); + } + for (size_t i=0; i<n_sparse_fear; ++i) { + nonZeroFeatures.push_back(nonZeroFeaturesFear[i]); + lossMinusModelScoreDiffs.push_back(diff/n); + } + cerr << "Rank " << rank << ", epoch " << epoch << ", core diff: " << diff/n << endl; + cerr << "Rank " << rank << ", epoch " << epoch << ", hope diff: " << ((diff/n)*1.1) << endl; + cerr << "Rank " << rank << ", epoch " << epoch << ", fear diff: " << diff/n << endl; + } + } + + assert(nonZeroFeatures.size() == lossMinusModelScoreDiffs.size()); + + // run optimisation: compute alphas for all given constraints + vector<float> alphas; + ScoreComponentCollection summedUpdate; + if (violatedConstraintsBefore > 0) { + cerr << "Rank " << rank << ", epoch " << epoch << ", number of constraints passed to optimizer: " << nonZeroFeatures.size() << endl; + alphas = Hildreth::optimise(nonZeroFeatures, lossMinusModelScoreDiffs, m_slack); + + // Update the weight vector according to the alphas and the feature value differences + // * w' = w' + SUM alpha_i * (h_i(oracle) - h_i(hypothesis)) + for (size_t k = 0; k < nonZeroFeatures.size(); ++k) { + float alpha = alphas[k]; + cerr << "Rank " << rank << ", epoch " << epoch << ", alpha: " << alpha << endl; + if (alpha != 0) { + ScoreComponentCollection update(nonZeroFeatures[k]); + update.MultiplyEquals(alpha); + + // sum updates + summedUpdate.PlusEquals(update); + } + } + } + else { + cerr << "Rank " << rank << ", epoch " << epoch << ", no constraint violated for this batch" << endl; + // return 0; + return 1; + } + + // apply learning rate + if (learning_rate != 1) { + cerr << "Rank " << rank << ", epoch " << epoch << ", apply learning rate " << learning_rate << " to update." << endl; + summedUpdate.MultiplyEquals(learning_rate); + } + + // scale update by BLEU of oracle (for batch size 1 only) + if (featureValuesHope.size() == 1) { + if (m_scale_update) { + cerr << "Rank " << rank << ", epoch " << epoch << ", scaling summed update with oracle bleu score " << bleuScoresHope[0][0] << endl; + summedUpdate.MultiplyEquals(bleuScoresHope[0][0]); + } + } + + //cerr << "Rank " << rank << ", epoch " << epoch << ", update: " << summedUpdate << endl; + weightUpdate.PlusEquals(summedUpdate); + return 0; +} + +size_t MiraOptimiser::updateWeightsHopeFearSummed( + Moses::ScoreComponentCollection& weightUpdate, + const std::vector< std::vector<Moses::ScoreComponentCollection> >& featureValuesHope, + const std::vector< std::vector<Moses::ScoreComponentCollection> >& featureValuesFear, + const std::vector<std::vector<float> >& bleuScoresHope, + const std::vector<std::vector<float> >& bleuScoresFear, + const std::vector<std::vector<float> >& modelScoresHope, + const std::vector<std::vector<float> >& modelScoresFear, + float learning_rate, + size_t rank, + size_t epoch, + bool rescaleSlack, + bool makePairs) { + + // vector of feature values differences for all created constraints + ScoreComponentCollection averagedFeatureDiffs; + float averagedViolations = 0; + + // Make constraints for new hypothesis translations + float epsilon = 0.0001; + int violatedConstraintsBefore = 0; + + if (!makePairs) { + ScoreComponentCollection featureValueDiff; + float lossHope = 0, lossFear = 0, modelScoreHope = 0, modelScoreFear = 0, hopeCount = 0, fearCount = 0; + // add all hope vectors + for (size_t i = 0; i < featureValuesHope.size(); ++i) { + for (size_t j = 0; j < featureValuesHope[i].size(); ++j) { + featureValueDiff.PlusEquals(featureValuesHope[i][j]); + lossHope += bleuScoresHope[i][j]; + modelScoreHope += modelScoresHope[i][j]; + ++hopeCount; + } + } + lossHope /= hopeCount; + modelScoreHope /= hopeCount; + + // subtract all fear vectors + for (size_t i = 0; i < featureValuesFear.size(); ++i) { + for (size_t j = 0; j < featureValuesFear[i].size(); ++j) { + featureValueDiff.MinusEquals(featureValuesFear[i][j]); + lossFear += bleuScoresFear[i][j]; + modelScoreFear += modelScoresFear[i][j]; + ++fearCount; + } + } + lossFear /= fearCount; + modelScoreFear /= fearCount; + + if (featureValueDiff.GetL1Norm() == 0) { + cerr << "Rank " << rank << ", epoch " << epoch << ", features equal --> skip" << endl; + cerr << "Rank " << rank << ", epoch " << epoch << ", no constraint violated for this batch" << endl; + return 1; + } + + // check if constraint is violated + float lossDiff = lossHope - lossFear; + float modelScoreDiff = modelScoreHope - modelScoreFear; + float diff = 0; + if (lossDiff > modelScoreDiff) + diff = lossDiff - modelScoreDiff; + if (diff > epsilon) + ++violatedConstraintsBefore; + cerr << "Rank " << rank << ", epoch " << epoch << ", constraint: " << modelScoreDiff << " >= " << lossDiff << " (current violation: " <<\ + diff << ")" << endl; + + // add constraint + averagedFeatureDiffs = featureValueDiff; + averagedViolations = diff; + } + else { + // iterate over input sentences (1 (online) or more (batch)) + for (size_t i = 0; i < featureValuesHope.size(); ++i) { + // Pick all pairs[j,j] of hope and fear translations for one input sentence and add them up + for (size_t j = 0; j < featureValuesHope[i].size(); ++j) { + ScoreComponentCollection featureValueDiff = featureValuesHope[i][j]; + featureValueDiff.MinusEquals(featureValuesFear[i][j]); + if (featureValueDiff.GetL1Norm() == 0) { + cerr << "Rank " << rank << ", epoch " << epoch << ", features equal --> skip" << endl; + continue; + } + + // check if constraint is violated + float lossDiff = bleuScoresHope[i][j] - bleuScoresFear[i][j]; + float modelScoreDiff = modelScoresHope[i][j] - modelScoresFear[i][j]; + if (rescaleSlack) { + cerr << "Rank " << rank << ", epoch " << epoch << ", modelScoreDiff scaled by lossDiff: " << modelScoreDiff << " --> " << modelScoreDiff*lossDiff << endl; + modelScoreDiff *= lossDiff; + } + float diff = 0; + if (lossDiff > modelScoreDiff) + diff = lossDiff - modelScoreDiff; + if (diff > epsilon) + ++violatedConstraintsBefore; + cerr << "Rank " << rank << ", epoch " << epoch << ", constraint: " << modelScoreDiff << " >= " << lossDiff << " (current violation: " << diff << ")" << endl; + + // add constraint + if (rescaleSlack) { + averagedFeatureDiffs.MultiplyEquals(lossDiff); + cerr << "Rank " << rank << ", epoch " << epoch << ", featureValueDiff scaled by lossDiff." << endl; + } + averagedFeatureDiffs.PlusEquals(featureValueDiff); + averagedViolations += diff; + } + } + } + + // divide by number of constraints (1/n) + if (!makePairs) { + averagedFeatureDiffs.DivideEquals(featureValuesHope[0].size()); + } + else { + averagedFeatureDiffs.DivideEquals(featureValuesHope[0].size()); + averagedViolations /= featureValuesHope[0].size(); + } + //cerr << "Rank " << rank << ", epoch " << epoch << ", averaged feature diffs: " << averagedFeatureDiffs << endl; + cerr << "Rank " << rank << ", epoch " << epoch << ", averaged violations: " << averagedViolations << endl; + + if (violatedConstraintsBefore > 0) { + // compute alpha for given constraint: (loss diff - model score diff) / || feature value diff ||^2 + // featureValueDiff.GetL2Norm() * featureValueDiff.GetL2Norm() == featureValueDiff.InnerProduct(featureValueDiff) + // from Crammer&Singer 2006: alpha = min {C , l_t/ ||x||^2} + // adjusted for 1 slack according to Joachims 2009, OP4 (margin rescaling), OP5 (slack rescaling) + float squaredNorm = averagedFeatureDiffs.GetL2Norm() * averagedFeatureDiffs.GetL2Norm(); + float alpha = averagedViolations / squaredNorm; + cerr << "Rank " << rank << ", epoch " << epoch << ", unclipped alpha: " << alpha << endl; + if (m_slack > 0 ) { + if (alpha > m_slack) { + alpha = m_slack; + } + else if (alpha < m_slack*(-1)) { + alpha = m_slack*(-1); + } + } + cerr << "Rank " << rank << ", epoch " << epoch << ", clipped alpha: " << alpha << endl; + + // compute update + averagedFeatureDiffs.MultiplyEquals(alpha); + weightUpdate.PlusEquals(averagedFeatureDiffs); + return 0; + } + else { + cerr << "Rank " << rank << ", epoch " << epoch << ", no constraint violated for this batch" << endl; + return 1; + } +} + +} + diff --git a/mira/MiraTest.cpp b/mira/MiraTest.cpp new file mode 100644 index 000000000..774b324f8 --- /dev/null +++ b/mira/MiraTest.cpp @@ -0,0 +1,24 @@ +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2010 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + + +//Supplies the main for the mira test module +#define BOOST_TEST_MODULE mira +#include <boost/test/unit_test.hpp> + diff --git a/mira/Optimiser.h b/mira/Optimiser.h new file mode 100644 index 000000000..213ee054e --- /dev/null +++ b/mira/Optimiser.h @@ -0,0 +1,174 @@ +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2010 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ +#ifndef _MIRA_OPTIMISER_H_ +#define _MIRA_OPTIMISER_H_ + +#include <vector> + +#include "ScoreComponentCollection.h" + + +namespace Mira { + + class Optimiser { + public: + Optimiser() {} + + virtual size_t updateWeightsHopeFear( + Moses::ScoreComponentCollection& weightUpdate, + const std::vector<std::vector<Moses::ScoreComponentCollection> >& featureValuesHope, + const std::vector<std::vector<Moses::ScoreComponentCollection> >& featureValuesFear, + const std::vector<std::vector<float> >& bleuScoresHope, + const std::vector<std::vector<float> >& bleuScoresFear, + const std::vector<std::vector<float> >& modelScoresHope, + const std::vector<std::vector<float> >& modelScoresFear, + float learning_rate, + size_t rank, + size_t epoch, + int updatePosition = -1) = 0; + }; + + class Perceptron : public Optimiser { + public: + virtual size_t updateWeightsHopeFear( + Moses::ScoreComponentCollection& weightUpdate, + const std::vector<std::vector<Moses::ScoreComponentCollection> >& featureValuesHope, + const std::vector<std::vector<Moses::ScoreComponentCollection> >& featureValuesFear, + const std::vector<std::vector<float> >& bleuScoresHope, + const std::vector<std::vector<float> >& bleuScoresFear, + const std::vector<std::vector<float> >& modelScoresHope, + const std::vector<std::vector<float> >& modelScoresFear, + float learning_rate, + size_t rank, + size_t epoch, + int updatePosition = -1); + }; + + class MiraOptimiser : public Optimiser { + public: + MiraOptimiser() : + Optimiser() { } + + MiraOptimiser( + float slack, bool scale_margin, bool scale_margin_precision, + bool scale_update, bool scale_update_precision, bool boost, bool normaliseMargin, float sigmoidParam) : + Optimiser(), + m_slack(slack), + m_scale_margin(scale_margin), + m_scale_margin_precision(scale_margin_precision), + m_scale_update(scale_update), + m_scale_update_precision(scale_update_precision), + m_precision(1), + m_boost(boost), + m_normaliseMargin(normaliseMargin), + m_sigmoidParam(sigmoidParam) { } + + size_t updateWeights( + Moses::ScoreComponentCollection& weightUpdate, + const std::vector<std::vector<Moses::ScoreComponentCollection> >& featureValues, + const std::vector<std::vector<float> >& losses, + const std::vector<std::vector<float> >& bleuScores, + const std::vector<std::vector<float> >& modelScores, + const std::vector< Moses::ScoreComponentCollection>& oracleFeatureValues, + const std::vector< float> oracleBleuScores, + const std::vector< float> oracleModelScores, + float learning_rate, + size_t rank, + size_t epoch); + virtual size_t updateWeightsHopeFear( + Moses::ScoreComponentCollection& weightUpdate, + const std::vector<std::vector<Moses::ScoreComponentCollection> >& featureValuesHope, + const std::vector<std::vector<Moses::ScoreComponentCollection> >& featureValuesFear, + const std::vector<std::vector<float> >& bleuScoresHope, + const std::vector<std::vector<float> >& bleuScoresFear, + const std::vector<std::vector<float> >& modelScoresHope, + const std::vector<std::vector<float> >& modelScoresFear, + float learning_rate, + size_t rank, + size_t epoch, + int updatePosition = -1); + size_t updateWeightsHopeFearSelective( + Moses::ScoreComponentCollection& weightUpdate, + const std::vector<std::vector<Moses::ScoreComponentCollection> >& featureValuesHope, + const std::vector<std::vector<Moses::ScoreComponentCollection> >& featureValuesFear, + const std::vector<std::vector<float> >& bleuScoresHope, + const std::vector<std::vector<float> >& bleuScoresFear, + const std::vector<std::vector<float> >& modelScoresHope, + const std::vector<std::vector<float> >& modelScoresFear, + float learning_rate, + size_t rank, + size_t epoch, + int updatePosition = -1); + size_t updateWeightsHopeFearSummed( + Moses::ScoreComponentCollection& weightUpdate, + const std::vector<std::vector<Moses::ScoreComponentCollection> >& featureValuesHope, + const std::vector<std::vector<Moses::ScoreComponentCollection> >& featureValuesFear, + const std::vector<std::vector<float> >& bleuScoresHope, + const std::vector<std::vector<float> >& bleuScoresFear, + const std::vector<std::vector<float> >& modelScoresHope, + const std::vector<std::vector<float> >& modelScoresFear, + float learning_rate, + size_t rank, + size_t epoch, + bool rescaleSlack, + bool makePairs); + size_t updateWeightsAnalytically( + Moses::ScoreComponentCollection& weightUpdate, + Moses::ScoreComponentCollection& featureValuesHope, + Moses::ScoreComponentCollection& featureValuesFear, + float bleuScoreHope, + float bleuScoreFear, + float modelScoreHope, + float modelScoreFear, + float learning_rate, + size_t rank, + size_t epoch); + + void setSlack(float slack) { + m_slack = slack; + } + + void setPrecision(float precision) { + m_precision = precision; + } + + private: + // regularise Hildreth updates + float m_slack; + + // scale margin with BLEU score or precision + bool m_scale_margin, m_scale_margin_precision; + + // scale update with oracle BLEU score or precision + bool m_scale_update, m_scale_update_precision; + + float m_precision; + + // boosting of updates on misranked candidates + bool m_boost; + + // squash margin between 0 and 1 (or depending on m_sigmoidParam) + bool m_normaliseMargin; + + // y=sigmoidParam is the axis that this sigmoid approaches + float m_sigmoidParam ; + }; +} + +#endif diff --git a/mira/Perceptron.cpp b/mira/Perceptron.cpp new file mode 100644 index 000000000..569a83216 --- /dev/null +++ b/mira/Perceptron.cpp @@ -0,0 +1,52 @@ +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2010 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#include "Optimiser.h" + +using namespace Moses; +using namespace std; + +namespace Mira { + +size_t Perceptron::updateWeightsHopeFear( + ScoreComponentCollection& weightUpdate, + const vector< vector<ScoreComponentCollection> >& featureValuesHope, + const vector< vector<ScoreComponentCollection> >& featureValuesFear, + const vector< vector<float> >& dummy1, + const vector< vector<float> >& dummy2, + const vector< vector<float> >& dummy3, + const vector< vector<float> >& dummy4, + float perceptron_learning_rate, + size_t rank, + size_t epoch, + int updatePosition) +{ + cerr << "Rank " << rank << ", epoch " << epoch << ", hope: " << featureValuesHope[0][0] << endl; + cerr << "Rank " << rank << ", epoch " << epoch << ", fear: " << featureValuesFear[0][0] << endl; + ScoreComponentCollection featureValueDiff = featureValuesHope[0][0]; + featureValueDiff.MinusEquals(featureValuesFear[0][0]); + cerr << "Rank " << rank << ", epoch " << epoch << ", hope - fear: " << featureValueDiff << endl; + featureValueDiff.MultiplyEquals(perceptron_learning_rate); + weightUpdate.PlusEquals(featureValueDiff); + cerr << "Rank " << rank << ", epoch " << epoch << ", update: " << featureValueDiff << endl; + return 0; +} + +} + diff --git a/mira/expt.cfg b/mira/expt.cfg new file mode 100644 index 000000000..416eb1d3f --- /dev/null +++ b/mira/expt.cfg @@ -0,0 +1,34 @@ +[general] +name=expt1 +moses-home=/path/to/moses/dir/ +cwd=/path/to/current/dir/ +working-dir=${cwd}/experiment +data-dir=${cwd}/data +decoder-settings=-mp -search-algorithm 1 -cube-pruning-pop-limit 5000 -s 5000 + +[train] +trainer=${moses-home}/mira/mira +input-file=${data-dir}/tune.input +reference-files=${data-dir}/tune.reference +moses-ini-file=${data-dir}/moses.ini +hours=48 +jobs=8 +slots=8 +epochs=10 +learner=mira +mixing-frequency=5 +weight-dump-frequency=1 +extra-args=--sentence-level-bleu 1 --hope-n 1 --fear-n 1 + +[devtest] +moses=${moses-home}/moses-cmd/src/moses +bleu=${moses-home}/scripts/generic/multi-bleu.perl +input-file=${data-dir}/devtest.input +reference-file=${data-dir}/devtest.reference +moses-ini-file=${data-dir}/moses.test.ini +hours=12 +extra-args= +skip-dev=1 +skip-devtest=0 +skip-submit=0 + diff --git a/mira/mira.xcodeproj/project.pbxproj b/mira/mira.xcodeproj/project.pbxproj new file mode 100644 index 000000000..67662f4e0 --- /dev/null +++ b/mira/mira.xcodeproj/project.pbxproj @@ -0,0 +1,401 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 45; + objects = { + +/* Begin PBXBuildFile section */ + 1E141A311243527800123194 /* Perceptron.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E141A2F1243527800123194 /* Perceptron.cpp */; }; + 1E56EBF51243B91600E8315C /* MiraOptimiser.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E56EBF41243B91600E8315C /* MiraOptimiser.cpp */; }; + 1E9DC63C1242602F0059001A /* Decoder.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E9DC6391242602F0059001A /* Decoder.cpp */; }; + 1E9DC63D1242602F0059001A /* Main.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 1E9DC63B1242602F0059001A /* Main.cpp */; }; + 1E9DC6DA1242684C0059001A /* libmoses-chart.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 1E9DC6D1124268310059001A /* libmoses-chart.a */; }; + 1E9DC6DB124268510059001A /* libmoses.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 1E9DC6CB124268270059001A /* libmoses.a */; }; + 1E9DC6DC124268580059001A /* libOnDiskPt.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 1E9DC6D9124268440059001A /* libOnDiskPt.a */; }; + 8DD76F6A0486A84900D96B5E /* mira.1 in CopyFiles */ = {isa = PBXBuildFile; fileRef = C6859E8B029090EE04C91782 /* mira.1 */; }; +/* End PBXBuildFile section */ + +/* Begin PBXContainerItemProxy section */ + 1E9DC6CA124268270059001A /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 1E9DC6C6124268270059001A /* moses.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = D2AAC046055464E500DB518D; + remoteInfo = moses; + }; + 1E9DC6D0124268310059001A /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 1E9DC6CC124268310059001A /* moses-chart.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = D2AAC046055464E500DB518D; + remoteInfo = "moses-chart"; + }; + 1E9DC6D8124268440059001A /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 1E9DC6D4124268440059001A /* OnDiskPt.xcodeproj */; + proxyType = 2; + remoteGlobalIDString = D2AAC046055464E500DB518D; + remoteInfo = OnDiskPt; + }; + 1EF4E84C12440612006233A0 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 1E9DC6C6124268270059001A /* moses.xcodeproj */; + proxyType = 1; + remoteGlobalIDString = D2AAC045055464E500DB518D /* moses */; + remoteInfo = moses; + }; + 1EF4E84E12440612006233A0 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 1E9DC6CC124268310059001A /* moses-chart.xcodeproj */; + proxyType = 1; + remoteGlobalIDString = D2AAC045055464E500DB518D /* moses-chart */; + remoteInfo = "moses-chart"; + }; + 1EF4E85012440612006233A0 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 1E9DC6D4124268440059001A /* OnDiskPt.xcodeproj */; + proxyType = 1; + remoteGlobalIDString = D2AAC045055464E500DB518D /* OnDiskPt */; + remoteInfo = OnDiskPt; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXCopyFilesBuildPhase section */ + 8DD76F690486A84900D96B5E /* CopyFiles */ = { + isa = PBXCopyFilesBuildPhase; + buildActionMask = 8; + dstPath = /usr/share/man/man1/; + dstSubfolderSpec = 0; + files = ( + 8DD76F6A0486A84900D96B5E /* mira.1 in CopyFiles */, + ); + runOnlyForDeploymentPostprocessing = 1; + }; +/* End PBXCopyFilesBuildPhase section */ + +/* Begin PBXFileReference section */ + 1E141A2F1243527800123194 /* Perceptron.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = Perceptron.cpp; sourceTree = "<group>"; }; + 1E56EBF41243B91600E8315C /* MiraOptimiser.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = MiraOptimiser.cpp; sourceTree = "<group>"; }; + 1E9DC6391242602F0059001A /* Decoder.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = Decoder.cpp; sourceTree = "<group>"; }; + 1E9DC63A1242602F0059001A /* Decoder.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Decoder.h; sourceTree = "<group>"; }; + 1E9DC63B1242602F0059001A /* Main.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = Main.cpp; sourceTree = "<group>"; }; + 1E9DC63E124260370059001A /* Optimiser.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Optimiser.h; sourceTree = "<group>"; }; + 1E9DC6C6124268270059001A /* moses.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; name = moses.xcodeproj; path = ../moses/moses.xcodeproj; sourceTree = SOURCE_ROOT; }; + 1E9DC6CC124268310059001A /* moses-chart.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; name = "moses-chart.xcodeproj"; path = "../moses-chart/moses-chart.xcodeproj"; sourceTree = SOURCE_ROOT; }; + 1E9DC6D4124268440059001A /* OnDiskPt.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; name = OnDiskPt.xcodeproj; path = ../OnDiskPt/OnDiskPt.xcodeproj; sourceTree = SOURCE_ROOT; }; + 1E9DC76712426FC60059001A /* Main.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Main.h; sourceTree = "<group>"; }; + 8DD76F6C0486A84900D96B5E /* mira */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = mira; sourceTree = BUILT_PRODUCTS_DIR; }; + C6859E8B029090EE04C91782 /* mira.1 */ = {isa = PBXFileReference; lastKnownFileType = text.man; path = mira.1; sourceTree = "<group>"; }; +/* End PBXFileReference section */ + +/* Begin PBXFrameworksBuildPhase section */ + 8DD76F660486A84900D96B5E /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 1E9DC6DC124268580059001A /* libOnDiskPt.a in Frameworks */, + 1E9DC6DB124268510059001A /* libmoses.a in Frameworks */, + 1E9DC6DA1242684C0059001A /* libmoses-chart.a in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + 08FB7794FE84155DC02AAC07 /* mira */ = { + isa = PBXGroup; + children = ( + 1E9DC6D4124268440059001A /* OnDiskPt.xcodeproj */, + 1E9DC6CC124268310059001A /* moses-chart.xcodeproj */, + 1E9DC6C6124268270059001A /* moses.xcodeproj */, + 08FB7795FE84155DC02AAC07 /* Source */, + C6859E8C029090F304C91782 /* Documentation */, + 1AB674ADFE9D54B511CA2CBB /* Products */, + ); + name = mira; + sourceTree = "<group>"; + }; + 08FB7795FE84155DC02AAC07 /* Source */ = { + isa = PBXGroup; + children = ( + 1E56EBF41243B91600E8315C /* MiraOptimiser.cpp */, + 1E141A2F1243527800123194 /* Perceptron.cpp */, + 1E9DC63E124260370059001A /* Optimiser.h */, + 1E9DC6391242602F0059001A /* Decoder.cpp */, + 1E9DC63A1242602F0059001A /* Decoder.h */, + 1E9DC63B1242602F0059001A /* Main.cpp */, + 1E9DC76712426FC60059001A /* Main.h */, + ); + name = Source; + sourceTree = "<group>"; + }; + 1AB674ADFE9D54B511CA2CBB /* Products */ = { + isa = PBXGroup; + children = ( + 8DD76F6C0486A84900D96B5E /* mira */, + ); + name = Products; + sourceTree = "<group>"; + }; + 1E9DC6C7124268270059001A /* Products */ = { + isa = PBXGroup; + children = ( + 1E9DC6CB124268270059001A /* libmoses.a */, + ); + name = Products; + sourceTree = "<group>"; + }; + 1E9DC6CD124268310059001A /* Products */ = { + isa = PBXGroup; + children = ( + 1E9DC6D1124268310059001A /* libmoses-chart.a */, + ); + name = Products; + sourceTree = "<group>"; + }; + 1E9DC6D5124268440059001A /* Products */ = { + isa = PBXGroup; + children = ( + 1E9DC6D9124268440059001A /* libOnDiskPt.a */, + ); + name = Products; + sourceTree = "<group>"; + }; + C6859E8C029090F304C91782 /* Documentation */ = { + isa = PBXGroup; + children = ( + C6859E8B029090EE04C91782 /* mira.1 */, + ); + name = Documentation; + sourceTree = "<group>"; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + 8DD76F620486A84900D96B5E /* mira */ = { + isa = PBXNativeTarget; + buildConfigurationList = 1DEB923108733DC60010E9CD /* Build configuration list for PBXNativeTarget "mira" */; + buildPhases = ( + 8DD76F640486A84900D96B5E /* Sources */, + 8DD76F660486A84900D96B5E /* Frameworks */, + 8DD76F690486A84900D96B5E /* CopyFiles */, + ); + buildRules = ( + ); + dependencies = ( + 1EF4E84D12440612006233A0 /* PBXTargetDependency */, + 1EF4E84F12440612006233A0 /* PBXTargetDependency */, + 1EF4E85112440612006233A0 /* PBXTargetDependency */, + ); + name = mira; + productInstallPath = "$(HOME)/bin"; + productName = mira; + productReference = 8DD76F6C0486A84900D96B5E /* mira */; + productType = "com.apple.product-type.tool"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + 08FB7793FE84155DC02AAC07 /* Project object */ = { + isa = PBXProject; + buildConfigurationList = 1DEB923508733DC60010E9CD /* Build configuration list for PBXProject "mira" */; + compatibilityVersion = "Xcode 3.1"; + hasScannedForEncodings = 1; + mainGroup = 08FB7794FE84155DC02AAC07 /* mira */; + projectDirPath = ""; + projectReferences = ( + { + ProductGroup = 1E9DC6CD124268310059001A /* Products */; + ProjectRef = 1E9DC6CC124268310059001A /* moses-chart.xcodeproj */; + }, + { + ProductGroup = 1E9DC6C7124268270059001A /* Products */; + ProjectRef = 1E9DC6C6124268270059001A /* moses.xcodeproj */; + }, + { + ProductGroup = 1E9DC6D5124268440059001A /* Products */; + ProjectRef = 1E9DC6D4124268440059001A /* OnDiskPt.xcodeproj */; + }, + ); + projectRoot = ""; + targets = ( + 8DD76F620486A84900D96B5E /* mira */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXReferenceProxy section */ + 1E9DC6CB124268270059001A /* libmoses.a */ = { + isa = PBXReferenceProxy; + fileType = archive.ar; + path = libmoses.a; + remoteRef = 1E9DC6CA124268270059001A /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 1E9DC6D1124268310059001A /* libmoses-chart.a */ = { + isa = PBXReferenceProxy; + fileType = archive.ar; + path = "libmoses-chart.a"; + remoteRef = 1E9DC6D0124268310059001A /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; + 1E9DC6D9124268440059001A /* libOnDiskPt.a */ = { + isa = PBXReferenceProxy; + fileType = archive.ar; + path = libOnDiskPt.a; + remoteRef = 1E9DC6D8124268440059001A /* PBXContainerItemProxy */; + sourceTree = BUILT_PRODUCTS_DIR; + }; +/* End PBXReferenceProxy section */ + +/* Begin PBXSourcesBuildPhase section */ + 8DD76F640486A84900D96B5E /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 1E9DC63C1242602F0059001A /* Decoder.cpp in Sources */, + 1E9DC63D1242602F0059001A /* Main.cpp in Sources */, + 1E141A311243527800123194 /* Perceptron.cpp in Sources */, + 1E56EBF51243B91600E8315C /* MiraOptimiser.cpp in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + 1EF4E84D12440612006233A0 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + name = moses; + targetProxy = 1EF4E84C12440612006233A0 /* PBXContainerItemProxy */; + }; + 1EF4E84F12440612006233A0 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + name = "moses-chart"; + targetProxy = 1EF4E84E12440612006233A0 /* PBXContainerItemProxy */; + }; + 1EF4E85112440612006233A0 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + name = OnDiskPt; + targetProxy = 1EF4E85012440612006233A0 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin XCBuildConfiguration section */ + 1DEB923208733DC60010E9CD /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + COPY_PHASE_STRIP = NO; + GCC_DYNAMIC_NO_PIC = NO; + GCC_ENABLE_FIX_AND_CONTINUE = YES; + GCC_MODEL_TUNING = G5; + GCC_OPTIMIZATION_LEVEL = 0; + INSTALL_PATH = /usr/local/bin; + LIBRARY_SEARCH_PATHS = ( + ../irstlm/lib/i386, + ../srilm/lib/macosx, + ); + OTHER_LDFLAGS = ( + "-lboost_program_options", + "-lz", + "-lirstlm", + "-lmisc", + "-ldstruct", + "-loolm", + "-lflm", + "-llattice", + ); + PRODUCT_NAME = mira; + }; + name = Debug; + }; + 1DEB923308733DC60010E9CD /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ALWAYS_SEARCH_USER_PATHS = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + GCC_MODEL_TUNING = G5; + INSTALL_PATH = /usr/local/bin; + LIBRARY_SEARCH_PATHS = ( + ../irstlm/lib/i386, + ../srilm/lib/macosx, + ); + OTHER_LDFLAGS = ( + "-lboost_program_options", + "-lz", + "-lirstlm", + "-lmisc", + "-ldstruct", + "-loolm", + "-lflm", + "-llattice", + ); + PRODUCT_NAME = mira; + }; + name = Release; + }; + 1DEB923608733DC60010E9CD /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ARCHS = "$(ARCHS_STANDARD_32_64_BIT)"; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_OPTIMIZATION_LEVEL = 0; + GCC_WARN_ABOUT_RETURN_TYPE = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + HEADER_SEARCH_PATHS = ( + /usr/local/include, + "../moses-chart/src", + ../moses/src, + ../irstlm/include, + ); + ONLY_ACTIVE_ARCH = YES; + PREBINDING = NO; + SDKROOT = macosx10.6; + }; + name = Debug; + }; + 1DEB923708733DC60010E9CD /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ARCHS = "$(ARCHS_STANDARD_32_64_BIT)"; + GCC_C_LANGUAGE_STANDARD = gnu99; + GCC_WARN_ABOUT_RETURN_TYPE = YES; + GCC_WARN_UNUSED_VARIABLE = YES; + HEADER_SEARCH_PATHS = ( + /usr/local/include, + "../moses-chart/src", + ../moses/src, + ../irstlm/include, + ); + PREBINDING = NO; + SDKROOT = macosx10.6; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + 1DEB923108733DC60010E9CD /* Build configuration list for PBXNativeTarget "mira" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 1DEB923208733DC60010E9CD /* Debug */, + 1DEB923308733DC60010E9CD /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 1DEB923508733DC60010E9CD /* Build configuration list for PBXProject "mira" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 1DEB923608733DC60010E9CD /* Debug */, + 1DEB923708733DC60010E9CD /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = 08FB7793FE84155DC02AAC07 /* Project object */; +} diff --git a/mira/training-expt.perl b/mira/training-expt.perl new file mode 100755 index 000000000..c143871b2 --- /dev/null +++ b/mira/training-expt.perl @@ -0,0 +1,994 @@ +#!/usr/bin/env perl + +use strict; +#eddie specific +use lib "/exports/informatics/inf_iccs_smt/perl/lib/perl5/site_perl"; +use Config::Simple; +use File::Basename; +use Getopt::Long "GetOptions"; + +my ($config_file,$execute,$continue); +die("training-expt.perl -config config-file [-exec]") + unless &GetOptions('config=s' => \$config_file, + 'cont=s' => \$continue, + 'exec' => \$execute); + +my $config = new Config::Simple($config_file) || + die "Error: unable to read config file \"$config_file\""; + +#substitution +foreach my $key ($config->param) { + my $value = $config->param($key); + while ($value =~ m/(.*?)\$\{(.*?)\}(.*)/) { + my $sub = $config->param($2); + if (! $sub) { + #try in this scope + my $scope = (split /\./, $key)[0]; + $sub = $config->param($scope . "." . $2); + } + if (! $sub) { + #then general + $sub = $config->param("general." . $2); + } + $value = $1 . $sub . $3; + } + print STDERR "$key => "; print STDERR $value; print STDERR "\n"; + $config->param($key,$value); +} + +#check if we're using sge +my $have_sge = 0; +if (`which qsub 2>/dev/null`) { + print "Using sge for job control.\n"; + $have_sge = 1; +} else { + print "No sge detected.\n"; +} + +#required global parameters +my $name = ¶m_required("general.name"); + +#optional globals +my $queue = ¶m("general.queue", "inf_iccs_smt"); +my $mpienv = ¶m("general.mpienv", "openmpi_smp8_mark2"); +my $vmem = ¶m("general.vmem", "6"); + +#wait for bleu files to appear in experiment folder if running as part of experiment.perl +my $wait_for_bleu = ¶m("general.wait-for-bleu", 0); + +#job control +my $jackknife = ¶m("general.jackknife", 0); +my $working_dir = ¶m("general.working-dir"); +my $general_decoder_settings = ¶m("general.decoder-settings", ""); +system("mkdir -p $working_dir") == 0 or die "Error: unable to create directory \"$working_dir\""; +my $train_script = "$name-train"; +my $job_name = "$name-t"; +my $hours = ¶m("train.hours",48); + +# check if we are tuning a meta feature +my $tuneMetaFeature = ¶m("general.tune-meta-feature", 0); +print STDERR "Tuning meta feature.. \n" if $tuneMetaFeature; + +# Check if a weight file with start weights was given +my $start_weight_file = ¶m("start.weightfile"); + +#required training parameters +my $singleRef = 1; +my ($moses_ini_file, $input_file, $reference_files); +my (@moses_ini_files_folds, @input_files_folds, @reference_files_folds); +if ($jackknife) { + my $array_ref = ¶m_required("train.moses-ini-files-folds"); + @moses_ini_files_folds= @$array_ref; + foreach my $ini (@moses_ini_files_folds) { + &check_exists ("moses ini file", $ini); + } + $array_ref = ¶m_required("train.input-files-folds"); + @input_files_folds = @$array_ref; + foreach my $in (@input_files_folds) { + &check_exists ("train input file", $in); + } + $array_ref = ¶m_required("train.reference-files-folds"); + @reference_files_folds = @$array_ref; + foreach my $ref (@reference_files_folds) { + &check_exists ("train reference file", $ref); + } +} +else { + $moses_ini_file = ¶m_required("train.moses-ini-file"); + &check_exists ("moses ini file", $moses_ini_file); + $input_file = ¶m_required("train.input-file"); + &check_exists ("train input file", $input_file); + $reference_files = ¶m_required("train.reference-files"); + if (&check_exists_noThrow ("ref files", $reference_files) != 0) { + for my $ref (glob $reference_files . "*") { + &check_exists ("ref files", $ref); + } + $singleRef = 0; + } +} + +# check if we want to continue an interrupted experiment +my $continue_expt = ¶m("general.continue-expt", 0); # number of experiment to continue +my $continue_epoch = 0; +if ($continue_expt > 0) { + die "ERROR: Continuing an experiment is not defined for tuning meta features.. \n\n" if ($tuneMetaFeature); + $continue_epoch = ¶m_required("general.continue-epoch", 0); + my $continue_weights = ¶m_required("general.continue-weights", 0); + open(CONT_WEIGHTS, $continue_weights); + my ($wp_weight, @pm_weights, $lm_weight, $lm2_weight, $d_weight, @lr_weights, %extra_weights); + my $num_core_weights = 0; + my $num_extra_weights = 0; + while(<CONT_WEIGHTS>) { + chomp; + my ($name,$value) = split; + next if ($name =~ /^!Unknown/); + next if ($name =~ /^BleuScore/); + next if ($name eq "DEFAULT_"); + if ($name eq "WordPenalty") { + $wp_weight = $value; + $num_core_weights += 1; + } elsif ($name =~ /^PhraseModel/) { + push @pm_weights,$value; + $num_core_weights += 1; + } elsif ($name =~ /^LM\:2/) { + $lm2_weight = $value; + $num_core_weights += 1; + } + elsif ($name =~ /^LM/) { + $lm_weight = $value; + $num_core_weights += 1; + } elsif ($name eq "Distortion") { + $d_weight = $value; + $num_core_weights += 1; + } elsif ($name =~ /^LexicalReordering/) { + push @lr_weights,$value; + $num_core_weights += 1; + } else { + $extra_weights{$name} = $value; + $num_extra_weights += 1; + } + } + close CONT_WEIGHTS; + print STDERR "num core weights to continue: $num_core_weights\n"; + print STDERR "num extra weights to continue: $num_extra_weights\n"; + + # write sparse weights to separate file + my $sparse_weights = $working_dir."/sparseWeights.expt".$continue_expt; + if ($num_extra_weights > 0) { + open(SPARSE, ">$sparse_weights"); + foreach my $name (sort keys %extra_weights) { + next if ($name eq "core"); + next if ($name eq "DEFAULT_"); + + # write only non-zero feature weights to file + if ($extra_weights{$name}) { + print SPARSE "$name $extra_weights{$name}\n"; + } + } + close SPARSE; + } + + # write new ini files with these weights + if ($jackknife) { + my @new_ini_files; + for (my $i=0; $i<=$#moses_ini_files_folds; $i++) { + my $ini_continue = $moses_ini_files_folds[$i].".continue".$continue_expt; + open(OLDINI, $moses_ini_files_folds[$i]); + open(NEWINI, ">$ini_continue"); + while(<OLDINI>) { + if (/weight-l/) { + print NEWINI "[weight-l]\n"; + print NEWINI $lm_weight; + print NEWINI "\n"; + + if (defined $lm2_weight) { + readline(OLDINI); + print NEWINI $lm2_weight; + print NEWINI "\n"; + } + + readline(OLDINI); + } elsif (/weight-t/) { + print NEWINI "[weight-t]\n"; + foreach my $pm_weight (@pm_weights) { + print NEWINI $pm_weight; + print NEWINI "\n"; + readline(OLDINI); + } + } elsif (/weight-d/) { + print NEWINI "[weight-d]\n"; + print NEWINI $d_weight; + print NEWINI "\n"; + readline(OLDINI); + foreach my $lr_weight (@lr_weights) { + print NEWINI $lr_weight; + print NEWINI "\n"; + readline(OLDINI); + } + } elsif (/weight-w/) { + print NEWINI "[weight-w]\n"; + print NEWINI $wp_weight; + print NEWINI "\n"; + readline(OLDINI); + } else { + print NEWINI; + } + } + if ($num_extra_weights > 0) { + print NEWINI "\n[weight-file]\n$sparse_weights\n"; + } + close OLDINI; + close NEWINI; + + print STDERR "new ini file: ".$ini_continue."\n"; + $moses_ini_files_folds[$i] = $ini_continue; + } + } + else { + my $ini_continue = $moses_ini_file.".continue".$continue_expt; + open(OLDINI, $moses_ini_file); + open(NEWINI, ">$ini_continue"); + while(<OLDINI>) { + if (/weight-l/) { + print NEWINI "[weight-l]\n"; + print NEWINI $lm_weight; + print NEWINI "\n"; + + if (defined $lm2_weight) { + readline(OLDINI); + print NEWINI $lm2_weight; + print NEWINI "\n"; + } + + readline(OLDINI); + } elsif (/weight-t/) { + print NEWINI "[weight-t]\n"; + foreach my $pm_weight (@pm_weights) { + print NEWINI $pm_weight; + print NEWINI "\n"; + readline(OLDINI); + } + } elsif (/weight-d/) { + print NEWINI "[weight-d]\n"; + print NEWINI $d_weight; + print NEWINI "\n"; + readline(OLDINI); + foreach my $lr_weight (@lr_weights) { + print NEWINI $lr_weight; + print NEWINI "\n"; + readline(OLDINI); + } + } elsif (/weight-w/) { + print NEWINI "[weight-w]\n"; + print NEWINI $wp_weight; + print NEWINI "\n"; + readline(OLDINI); + } + else { + print NEWINI; + } + } + if ($num_extra_weights > 0) { + print NEWINI "\n[weight-file]\n$sparse_weights\n"; + } + close OLDINI; + close NEWINI; + print STDERR "new ini file: ".$ini_continue."\n"; + $moses_ini_file = $ini_continue; + } +} + +my $trainer_exe = ¶m_required("train.trainer"); +&check_exists("Training executable", $trainer_exe); +#my $weights_file = ¶m_required("train.weights-file"); +#&check_exists("weights file ", $weights_file); + +#optional training parameters +my $epochs = ¶m("train.epochs"); +my $learner = ¶m("train.learner", "mira"); +my $batch = ¶m("train.batch", 1); # don't print this param twice (when printing training file) +my $extra_args = ¶m("train.extra-args"); +my $by_node = ¶m("train.by-node"); +my $slots = ¶m("train.slots",8); +my $jobs = ¶m("train.jobs",8); +my $mixing_frequency = ¶m("train.mixing-frequency", 1); # don't print this param twice +my $weight_dump_frequency = ¶m("train.weight-dump-frequency", 1); # don't print this param twice +my $burn_in = ¶m("train.burn-in"); +my $burn_in_input_file = ¶m("train.burn-in-input-file"); +my $burn_in_reference_files = ¶m("train.burn-in-reference-files"); +my $skipTrain = ¶m("train.skip"); +my $train_decoder_settings = ¶m("train.decoder-settings", ""); +if (!$train_decoder_settings) { + $train_decoder_settings = $general_decoder_settings; +} + +#devtest configuration +my ($devtest_input_file, $devtest_reference_files,$devtest_ini_file,$bleu_script,$use_moses); +my $test_exe = ¶m("devtest.moses"); +&check_exists("test executable", $test_exe); +$bleu_script = ¶m_required("devtest.bleu"); +&check_exists("multi-bleu script", $bleu_script); +$devtest_input_file = ¶m_required("devtest.input-file"); +&check_exists ("devtest input file", $devtest_input_file); +$devtest_reference_files = ¶m_required("devtest.reference-file"); +if (&check_exists_noThrow ("devtest ref file", $devtest_reference_files) != 0) { + for my $ref (glob $devtest_reference_files . "*") { + &check_exists ("devtest ref file", $ref); + } +} +$devtest_ini_file = ¶m_required("devtest.moses-ini-file"); +&check_exists ("devtest ini file", $devtest_ini_file); + + +my $weight_file_stem = "$name-weights"; +my $extra_memory_devtest = ¶m("devtest.extra-memory",0); +my $skip_devtest = ¶m("devtest.skip-devtest",0); +my $skip_dev = ¶m("devtest.skip-dev",0); +my $skip_submit_test = ¶m("devtest.skip-submit",0); +my $devtest_decoder_settings = ¶m("devtest.decoder-settings", ""); +if (!$devtest_decoder_settings) { + $devtest_decoder_settings = $general_decoder_settings; +} + + +# check that number of jobs, dump frequency and number of input sentences are compatible +# shard size = number of input sentences / number of jobs, ensure shard size >= dump frequency +if ($jackknife) { + # TODO.. +} +else { + my $result = `wc -l $input_file`; + my @result = split(/\s/, $result); + my $inputSize = $result[0]; + my $shardSize = $inputSize / $jobs; + if ($mixing_frequency) { + if ($shardSize < $mixing_frequency) { + $mixing_frequency = int($shardSize); + if ($mixing_frequency == 0) { + $mixing_frequency = 1; + } + + print STDERR "Warning: mixing frequency must not be larger than shard size, setting mixing frequency to $mixing_frequency\n"; + } + } + + if ($weight_dump_frequency != 0) { + if ($shardSize < $weight_dump_frequency) { + $weight_dump_frequency = int($shardSize); + if ($weight_dump_frequency == 0) { + $weight_dump_frequency = 1; + } + + print STDERR "Warning: weight dump frequency must not be larger than shard size, setting weight dump frequency to $weight_dump_frequency\n"; + } + } + + if ($mixing_frequency != 0) { + if ($mixing_frequency > ($shardSize/$batch)) { + $mixing_frequency = int($shardSize/$batch); + if ($mixing_frequency == 0) { + $mixing_frequency = 1; + } + + print STDERR "Warning: mixing frequency must not be larger than (shard size/batch size), setting mixing frequency to $mixing_frequency\n"; + } + } + + if ($weight_dump_frequency != 0) { + if ($weight_dump_frequency > ($shardSize/$batch)) { + $weight_dump_frequency = int($shardSize/$batch); + if ($weight_dump_frequency == 0) { + $weight_dump_frequency = 1; + } + + print STDERR "Warning: weight dump frequency must not be larger than (shard size/batch size), setting weight dump frequency to $weight_dump_frequency\n"; + } + } +} + +#file names +my $train_script_file = $working_dir . "/" . $train_script . ".sh"; +my $train_out = $train_script . ".out"; +my $train_err = $train_script . ".err"; +my $train_job_id = 0; + +my @refs; +if (ref($reference_files) eq 'ARRAY') { + @refs = @$reference_files; +} elsif ($singleRef){ + $refs[0] = $reference_files; +} else { + @refs = glob $reference_files . "*" +} +my $arr_refs = \@refs; + +if (!$skipTrain) { + #write the script + open TRAIN, ">$train_script_file" or die "Unable to open \"$train_script_file\" for writing"; + + &header(*TRAIN,$job_name,$working_dir,$slots,$jobs,$hours,$vmem,$train_out,$train_err); + if ($jobs == 1) { + print TRAIN "$trainer_exe "; + } + else { + if ($by_node) { + print TRAIN "mpirun -np $jobs --bynode $trainer_exe \\\n"; + } + else { + print TRAIN "mpirun -np \$NSLOTS $trainer_exe \\\n"; + } + } + + if ($jackknife) { + foreach my $ini (@moses_ini_files_folds) { + print TRAIN "--configs-folds $ini "; + } + print TRAIN "\\\n"; + foreach my $in (@input_files_folds) { + print TRAIN "--input-files-folds $in "; + } + print TRAIN "\\\n"; + for my $ref (@reference_files_folds) { + print TRAIN "--reference-files-folds $ref "; + } + print TRAIN "\\\n"; + } + else { + print TRAIN "-f $moses_ini_file \\\n"; + print TRAIN "-i $input_file \\\n"; + for my $ref (@refs) { + print TRAIN "-r $ref "; + } + print TRAIN "\\\n"; + } + if ($continue_epoch > 0) { + print TRAIN "--continue-epoch $continue_epoch \\\n"; + } + if ($burn_in) { + print TRAIN "--burn-in 1 \\\n"; + print TRAIN "--burn-in-input-file $burn_in_input_file \\\n"; + my @burnin_refs; + if (ref($burn_in_reference_files) eq 'ARRAY') { + @burnin_refs = @$burn_in_reference_files; + } else { + @burnin_refs = glob $burn_in_reference_files . "*"; # TODO: + } + for my $burnin_ref (@burnin_refs) { + &check_exists("burn-in ref file", $burnin_ref); + print TRAIN "--burn-in-reference-files $burnin_ref "; + } + print TRAIN "\\\n"; + } +#if ($weights_file) { +# print TRAIN "-w $weights_file \\\n"; +#} + if (defined $start_weight_file) { + print TRAIN "--start-weights $start_weight_file \\\n"; + } + print TRAIN "-l $learner \\\n"; + print TRAIN "--weight-dump-stem $weight_file_stem \\\n"; + print TRAIN "--mixing-frequency $mixing_frequency \\\n" if ($extra_args !~ /--mixing-frequency /); + print TRAIN "--weight-dump-frequency $weight_dump_frequency \\\n" if ($extra_args !~ /--weight-dump-frequency /); + print TRAIN "--epochs $epochs \\\n" if $epochs; + print TRAIN "--batch-size $batch \\\n" if ($extra_args !~ /--batch-size / && $extra_args !~ /-b /); + print TRAIN $extra_args." \\\n"; + print TRAIN "--decoder-settings \"$train_decoder_settings\" \\\n"; + if ($jobs == 1) { + print TRAIN "echo \"mira finished.\"\n"; + } + else { + print TRAIN "echo \"mpirun finished.\"\n"; + } + close TRAIN; + + if (! $execute) { + print STDERR "Written train file: $train_script_file\n"; + exit 0; + } + + #submit the training job + if ($have_sge) { + $train_job_id = &submit_job_sge($train_script_file); + + } else { + $train_job_id = &submit_job_no_sge($train_script_file, $train_out,$train_err); + } + + die "Failed to submit training job" unless $train_job_id; +} + +#wait for the next weights file to appear, or the training job to end +my $train_iteration = -1; +if ($continue_epoch > 0) { + $train_iteration += ($continue_epoch*$weight_dump_frequency); + print STDERR "Start from training iteration ".$train_iteration." instead of -1.\n"; +} +else { + print STDERR "Start from training iteration ".$train_iteration."\n"; +} + +while(1) { + my($epoch, $epoch_slice); + $train_iteration += 1; # starts at 0 + my $new_weight_file = "$working_dir/$weight_file_stem" . "_"; + if ($weight_dump_frequency == 0) { + print STDERR "No weights, no testing..\n"; + exit(0); + } + + #my $epoch = 1 + int $train_iteration / $weight_dump_frequency; + $epoch = int $train_iteration / $weight_dump_frequency; + $epoch_slice = $train_iteration % $weight_dump_frequency; + if ($weight_dump_frequency == 1) { + if ($train_iteration < 10) { + $new_weight_file .= "0".$train_iteration; + } + else { + $new_weight_file .= $train_iteration; + } + } else { + if ($epoch < 10) { + $new_weight_file .= "0".$epoch."_".$epoch_slice; + } + else { + $new_weight_file .= $epoch."_".$epoch_slice; + } + } + + print STDERR "Current epoch: ".$epoch."\n"; + my $expected_num_files = $epoch*$weight_dump_frequency; + if ($wait_for_bleu) { + print STDERR "Expected number of BLEU files: $expected_num_files \n"; + } + if (-e "$working_dir/stopping") { + wait_for_bleu($expected_num_files, $working_dir) if ($wait_for_bleu); + print STDERR "Training finished at " . scalar(localtime()) . " because stopping criterion was reached.\n"; + exit 0; + } + elsif (-e "$working_dir/finished") { + wait_for_bleu($expected_num_files, $working_dir) if ($wait_for_bleu); + print STDERR "Training finished at " . scalar(localtime()) . " because maximum number of epochs was reached.\n"; + exit 0; + } + else { + print STDERR "Waiting for $new_weight_file\n"; + if (!$skipTrain) { + while ((! -e $new_weight_file) && &check_running($train_job_id)) { + sleep 10; + } + } + if (! -e $new_weight_file ) { + if (-e "$working_dir/stopping") { + wait_for_bleu($expected_num_files, $working_dir) if ($wait_for_bleu); + print STDERR "Training finished at " . scalar(localtime()) . " because stopping criterion was reached.\n"; + exit 0; + } + elsif (-e "$working_dir/finished") { + wait_for_bleu($expected_num_files, $working_dir) if ($wait_for_bleu); + print STDERR "Training finished at " . scalar(localtime()) . " because maximum number of epochs was reached.\n"; + exit 0; + } + else { + # training finished with error + print STDERR "Error: training was aborted at " . scalar(localtime()) . "\n"; + exit 1; + } + } + } + + #new weight file written. create test script and submit + my $suffix = ""; + print STDERR "weight file exists? ".(-e $new_weight_file)."\n"; + if (!$skip_devtest) { + createTestScriptAndSubmit($epoch, $epoch_slice, $new_weight_file, $suffix, "devtest", $devtest_ini_file, $devtest_input_file, $devtest_reference_files, $skip_submit_test); + + my $regularized_weight_file = $new_weight_file."_reg"; + if (-e $regularized_weight_file) { + print STDERR "Submitting test script for regularized weights. \n"; + $epoch_slice .= "_reg"; + createTestScriptAndSubmit($epoch, $epoch_slice, $regularized_weight_file, $suffix, "devtest", $devtest_ini_file, $devtest_input_file, $devtest_reference_files, $skip_submit_test); + } + } + if (!$skip_dev) { + createTestScriptAndSubmit($epoch, $epoch_slice, $new_weight_file, $suffix, "dev", $moses_ini_file, $input_file, $reference_files, $skip_submit_test); + } +} + +sub wait_for_bleu() { + my $expected_num_files = $_[0]; + my $working_dir = $_[1]; + print STDERR "Waiting for $expected_num_files bleu files..\n"; + print STDERR "Path: $working_dir/*.bleu \n"; + my @bleu_files = glob("$working_dir/*.bleu"); + while (scalar(@bleu_files) < $expected_num_files) { + sleep 30; + @bleu_files = glob("$working_dir/*.bleu"); + print STDERR "currently have ".(scalar(@bleu_files))."\n"; + } + print STDERR "$expected_num_files BLEU files completed, continue.\n"; +} + +sub createTestScriptAndSubmit { + my $epoch = $_[0]; + my $epoch_slice = $_[1]; + my $new_weight_file = $_[2]; + my $suffix = $_[3]; + my $testtype = $_[4]; + my $old_ini_file = $_[5]; + my $input_file = $_[6]; + my $reference_file = $_[7]; + my $skip_submit = $_[8]; + + #file names + my $output_file; + my $output_error_file; + my $bleu_file; + my $file_id = ""; + if ($weight_dump_frequency == 1) { + if ($train_iteration < 10) { + $output_file = $working_dir."/".$name."_0".$train_iteration.$suffix."_$testtype".".out"; + $output_error_file = $working_dir."/".$name."_0".$train_iteration.$suffix."_$testtype".".err"; + $bleu_file = $working_dir."/".$name."_0".$train_iteration.$suffix."_$testtype".".bleu"; + $file_id = "0".$train_iteration.$suffix; + } + else { + $output_file = $working_dir."/".$name."_".$train_iteration.$suffix."_$testtype".".out"; + $output_error_file = $working_dir."/".$name."_".$train_iteration.$suffix."_$testtype".".err"; + $bleu_file = $working_dir."/".$name."_".$train_iteration.$suffix."_$testtype".".bleu"; + $file_id = $train_iteration.$suffix; + } + } + else { + if ($epoch < 10) { + $output_file = $working_dir."/".$name."_0".$epoch."_".$epoch_slice.$suffix."_$testtype".".out"; + $output_error_file = $working_dir."/".$name."_0".$epoch."_".$epoch_slice.$suffix."_$testtype".".err"; + $bleu_file = $working_dir."/".$name."_0".$epoch."_".$epoch_slice.$suffix."_$testtype".".bleu"; + $file_id = "0".$epoch."_".$epoch_slice.$suffix; + } + else { + $output_file = $working_dir."/".$name."_".$epoch."_".$epoch_slice.$suffix."_$testtype".".out"; + $output_error_file = $working_dir."/".$name."_".$epoch."_".$epoch_slice.$suffix."_$testtype".".err"; + $bleu_file = $working_dir."/".$name."_".$epoch."_".$epoch_slice.$suffix."_$testtype".".bleu"; + $file_id = $epoch."_".$epoch_slice.$suffix; + } + } + + my $job_name = $name."_".$testtype."_".$file_id; + + my $test_script = "$name-$testtype"; + my $test_script_file = "$working_dir/$test_script.$file_id.sh"; + my $test_out = "$test_script.$file_id.out"; + my $test_err = "$test_script.$file_id.err"; + + if (! (open TEST, ">$test_script_file" )) { + die "Unable to create test script $test_script_file\n"; + } + + my $hours = ¶m("test.hours",12); + my $extra_args = ¶m("test.extra-args"); + + # Splice the weights into the moses ini file. + my ($default_weight,$wordpenalty_weight,@phrasemodel_weights,$lm_weight,$lm2_weight,$distortion_weight,@lexicalreordering_weights); + + if (! (open WEIGHTS, "$new_weight_file")) { + die "Unable to open weights file $new_weight_file\n"; + } + + my $readCoreWeights = 0; + my $readExtraWeights = 0; + my %extra_weights; + my $abs_weights = 0; + my $metaFeature_wt_weight = 0; + my $metaFeature_pp_weight = 0; + while(<WEIGHTS>) { + chomp; + my ($name,$value) = split; + next if ($name =~ /^!Unknown/); + next if ($name =~ /^BleuScore/); + if ($name eq "DEFAULT_") { + $default_weight = $value; + } else { + if ($name eq "WordPenalty") { + $wordpenalty_weight = $value; + $abs_weights += abs($value); + $readCoreWeights += 1; + } elsif ($name =~ /^PhraseModel/) { + push @phrasemodel_weights,$value; + $abs_weights += abs($value); + $readCoreWeights += 1; + } elsif ($name =~ /^LM\:2/) { + $lm2_weight = $value; + $abs_weights += abs($value); + $readCoreWeights += 1; + } + elsif ($name =~ /^LM/) { + $lm_weight = $value; + $abs_weights += abs($value); + $readCoreWeights += 1; + } elsif ($name eq "Distortion") { + $distortion_weight = $value; + $abs_weights += abs($value); + $readCoreWeights += 1; + } elsif ($name =~ /^LexicalReordering/) { + push @lexicalreordering_weights,$value; + $abs_weights += abs($value); + $readCoreWeights += 1; + } elsif ($name =~ /^MetaFeature_wt/) { + $metaFeature_wt_weight = $value; + $abs_weights += abs($value); + $readCoreWeights += 1; + } + elsif ($name =~ /^MetaFeature_pp/) { + $metaFeature_pp_weight = $value; + $abs_weights += abs($value); + $readCoreWeights += 1; + } + else { + $extra_weights{$name} = $value; + $readExtraWeights += 1; + } + } + } + close WEIGHTS; + + print STDERR "Number of core weights read: ".$readCoreWeights."\n"; + print STDERR "Number of extra weights read: ".$readExtraWeights."\n"; + + # Create new ini file (changing format: expt1-devtest.00_2.ini instead of expt1-devtest.3.ini) + # my $new_ini_file = $working_dir."/".$test_script.".".$train_iteration.$suffix.".ini"; + my $new_ini_file = "$working_dir/$test_script.$file_id.ini"; + if (! (open NEWINI, ">$new_ini_file" )) { + die "Unable to create ini file $new_ini_file\n"; + } + if (! (open OLDINI, "$old_ini_file" )) { + die "Unable to read ini file $old_ini_file\n"; + } + + # write normalized weights to ini file + while(<OLDINI>) { + if (/weight-l/) { + print NEWINI "[weight-l]\n"; + print NEWINI ($lm_weight/$abs_weights); + print NEWINI "\n"; + + if (defined $lm2_weight) { + readline(OLDINI); + print NEWINI ($lm2_weight/$abs_weights); + print NEWINI "\n"; + } + + readline(OLDINI); + } elsif (/weight-t/) { + print NEWINI "[weight-t]\n"; + foreach my $phrasemodel_weight (@phrasemodel_weights) { + print NEWINI ($phrasemodel_weight/$abs_weights); + print NEWINI "\n"; + readline(OLDINI); + } + } elsif (/weight-d/) { + print NEWINI "[weight-d]\n"; + print NEWINI ($distortion_weight/$abs_weights); + print NEWINI "\n"; + readline(OLDINI); + foreach my $lexicalreordering_weight (@lexicalreordering_weights) { + print NEWINI ($lexicalreordering_weight/$abs_weights); + print NEWINI "\n"; + readline(OLDINI); + } + } elsif (/weight-wt/) { + print NEWINI "[weight-wt]\n"; + print NEWINI $metaFeature_wt_weight/$abs_weights; + print NEWINI "\n"; + readline(OLDINI); + } elsif (/weight-pp/) { + print NEWINI "[weight-pp]\n"; + print NEWINI $metaFeature_pp_weight/$abs_weights; + print NEWINI "\n"; + readline(OLDINI); + } + elsif (/weight-w/) { + print NEWINI "[weight-w]\n"; + print NEWINI ($wordpenalty_weight/$abs_weights); + print NEWINI "\n"; + readline(OLDINI); + } + else { + print NEWINI; + } + } + close OLDINI; + + my $writtenExtraWeights = 0; + + # if there are any non-core weights, write them to a weights file (normalized) + my $extra_weight_file = undef; + if (%extra_weights && !$tuneMetaFeature) { + $extra_weight_file = "$new_weight_file.sparse.scaled"; + if (! (open EXTRAWEIGHT,">$extra_weight_file")) { + print "Warning: unable to create extra weights file $extra_weight_file"; + next; + } +# my $core_weight = 1; +# if ($have_core) { +# $default_weight = $extra_weights{"DEFAULT_"}; +# $core_weight = $extra_weights{"core"}; +# } + foreach my $name (sort keys %extra_weights) { + next if ($name eq "core"); + next if ($name eq "DEFAULT_"); + my $value = $extra_weights{$name}/$abs_weights; + + # write only non-zero feature weights to file + if ($value) { +# $value /= $core_weight; + print EXTRAWEIGHT "$name $value\n"; + $writtenExtraWeights += 1; + } + } + } + + # add specification of sparse weight file to ini + if (!$tuneMetaFeature) { + print NEWINI "\n[weight-file] \n"; + print NEWINI "$extra_weight_file \n"; + close NEWINI; + } + + print TEST "#!/bin/sh\n"; + print TEST "#\$ -N $job_name\n"; + print TEST "#\$ -wd $working_dir\n"; + print TEST "#\$ -l h_rt=$hours:00:00\n"; + print TEST "#\$ -o $test_out\n"; + print TEST "#\$ -e $test_err\n"; + print TEST "\n"; + if ($have_sge) { +# some eddie specific stuff + print TEST ". /etc/profile.d/modules.sh\n"; + print TEST "module load openmpi/ethernet/gcc/latest\n"; + print TEST "export LD_LIBRARY_PATH=/exports/informatics/inf_iccs_smt/shared/boost/lib:\$LD_LIBRARY_PATH\n"; + } + print TEST "$test_exe $devtest_decoder_settings -i $input_file -f $new_ini_file "; +# now written to ini file +# if ($extra_weight_file) { +# print TEST "-weight-file $extra_weight_file "; +# } + print TEST $extra_args; + print TEST " 1> $output_file 2> $output_error_file\n"; + print TEST "echo \"Decoding of ".$testtype." set finished.\"\n"; + print TEST "$bleu_script $reference_file < $output_file > $bleu_file\n"; + print TEST "echo \"Computed BLEU score of ".$testtype." set.\"\n"; + close TEST; + + #launch testing + if(!$skip_submit) { + if ($have_sge) { + if ($extra_memory_devtest) { + print STDERR "Extra memory for test job: $extra_memory_devtest \n"; + &submit_job_sge_extra_memory($test_script_file,$extra_memory_devtest); + } + else { + &submit_job_sge($test_script_file); + } + } else { + &submit_job_no_sge($test_script_file, $test_out,$test_err); + } + } +} + +sub param { + my ($key,$default) = @_; + my $value = $config->param($key); + $value = $default if !$value; + # Empty arguments get interpreted as arrays + $value = 0 if (ref($value) eq 'ARRAY' && scalar(@$value) == 0); + return $value; +} + +sub param_required { + my ($key) = @_; + my $value = ¶m($key); + die "Error: required parameter \"$key\" was missing" if (!defined($value)); + #$value = join $value if (ref($value) eq 'ARRAY'); + return $value; +} + +sub header { + my ($OUT,$name,$working_dir,$slots,$jobs,$hours,$vmem,$out,$err) = @_; + print $OUT "#!/bin/sh\n"; + if ($have_sge) { + print $OUT "#\$ -N $name\n"; + print $OUT "#\$ -wd $working_dir\n"; + if ($jobs != 1) { + print $OUT "#\$ -pe $mpienv $slots\n"; + } + print $OUT "#\$ -l h_rt=$hours:00:00\n"; + print $OUT "#\$ -l h_vmem=$vmem" . "G" . "\n"; + print $OUT "#\$ -o $out\n"; + print $OUT "#\$ -e $err\n"; + } else { + print $OUT "\nNSLOTS=$jobs\n"; + } + print $OUT "\n"; + if ($have_sge) { +# some eddie specific stuff + print $OUT ". /etc/profile.d/modules.sh\n"; + print $OUT "module load openmpi/ethernet/gcc/latest\n"; + print $OUT "export LD_LIBRARY_PATH=/exports/informatics/inf_iccs_smt/shared/boost/lib:\$LD_LIBRARY_PATH\n"; + } +} + +sub check_exists { + my ($name,$filename) = @_; + die "Error: unable to read $name: \"$filename\"" if ! -r $filename; +} + +sub check_exists_noThrow { + my ($name,$filename) = @_; + return 1 if ! -r $filename; + return 0; +} + +# +# Used to submit train/test jobs. Return the job id, or 0 on failure +# + +sub submit_job_sge { + my($script_file) = @_; + my $qsub_result = `qsub -P $queue $script_file`; + print STDERR "SUBMIT CMD: qsub -P $queue $script_file\n"; + if ($qsub_result !~ /Your job (\d+)/) { + print STDERR "Failed to qsub job: $qsub_result\n"; + return 0; + } + my $job_name = basename($script_file); + print STDERR "Submitted job: $job_name id: $1 " . + scalar(localtime()) . "\n"; + return $1; +} + +sub submit_job_sge_extra_memory { + my($script_file,$extra_memory) = @_; + my $qsub_result = `qsub -pe $extra_memory -P $queue $script_file`; + print STDERR "SUBMIT CMD: qsub -pe $extra_memory -P $queue $script_file \n"; + if ($qsub_result !~ /Your job (\d+)/) { + print STDERR "Failed to qsub job: $qsub_result\n"; + return 0; + } + my $job_name = basename($script_file); + print STDERR "Submitted job: $job_name id: $1 " . + scalar(localtime()) . "\n"; + return $1; +} + +# +# As above, but no sge version. Returns the pid. +# + +sub submit_job_no_sge { + my($script_file,$out,$err) = @_; + my $pid = undef; + if ($pid = fork) { + my $job_name = basename($script_file); + print STDERR "Launched : $job_name pid: $pid " . scalar(localtime()) . "\n"; + return $pid; + } elsif (defined $pid) { + print STDERR "Executing script $script_file, writing to $out and $err.\n"; + `cd $working_dir; sh $script_file 1>$out 2> $err`; + exit; + } else { + # Fork failed + return 0; + } +} + +sub check_running { + my ($job_id) = @_; + if ($have_sge) { + return `qstat | grep $job_id`; + } else { + return `ps -e | grep $job_id | grep -v defunct`; + } +} + + + diff --git a/misc/queryPhraseTable.cpp b/misc/queryPhraseTable.cpp index 053d932ef..02e1c29a1 100644 --- a/misc/queryPhraseTable.cpp +++ b/misc/queryPhraseTable.cpp @@ -63,16 +63,16 @@ int main(int argc, char **argv) } else { for(uint i = 0; i < tgtcands.size(); i++) { std::cout << line << " |||"; - for(uint j = 0; j < tgtcands[i].first.size(); j++) - std::cout << ' ' << *tgtcands[i].first[j]; + for(uint j = 0; j < tgtcands[i].tokens.size(); j++) + std::cout << ' ' << *tgtcands[i].tokens[j]; std::cout << " |||"; if(useAlignments) { std::cout << " " << wordAlignment[i] << " |||"; } - for(uint j = 0; j < tgtcands[i].second.size(); j++) - std::cout << ' ' << tgtcands[i].second[j]; + for(uint j = 0; j < tgtcands[i].scores.size(); j++) + std::cout << ' ' << tgtcands[i].scores[j]; std::cout << '\n'; } std::cout << '\n'; diff --git a/moses-chart-cmd/src/IOWrapper.cpp b/moses-chart-cmd/src/IOWrapper.cpp index 766bb5e6f..f8ae9c51c 100644 --- a/moses-chart-cmd/src/IOWrapper.cpp +++ b/moses-chart-cmd/src/IOWrapper.cpp @@ -46,6 +46,9 @@ POSSIBILITY OF SUCH DAMAGE. #include "ChartTranslationOptions.h" #include "ChartHypothesis.h" +#include <boost/algorithm/string.hpp> +#include "FeatureVector.h" + using namespace std; using namespace Moses; @@ -142,6 +145,7 @@ InputType*IOWrapper::GetInput(InputType* inputType) } } + /*** * print surface factor only for the given phrase */ @@ -400,7 +404,7 @@ void IOWrapper::OutputNBestList(const ChartTrellisPathList &nBestList, const Cha // print the surface factor of the translation out << translationId << " ||| "; OutputSurface(out, outputPhrase, m_outputFactorOrder, false); - out << " |||"; + out << " ||| "; // print the scores in a hardwired order // before each model type, the corresponding command-line-like name must be emitted @@ -417,26 +421,29 @@ void IOWrapper::OutputNBestList(const ChartTrellisPathList &nBestList, const Cha } } - std::string lastName = ""; + // output stateful sparse features + const vector<const StatefulFeatureFunction*>& sff = system->GetStatefulFeatureFunctions(); + for( size_t i=0; i<sff.size(); i++ ) + if (sff[i]->GetNumScoreComponents() == ScoreProducer::unlimited) + OutputSparseFeatureScores( out, path, sff[i], lastName ); + // translation components const vector<PhraseDictionaryFeature*>& pds = system->GetPhraseDictionaries(); if (pds.size() > 0) { - for( size_t i=0; i<pds.size(); i++ ) { - size_t pd_numinputscore = pds[i]->GetNumInputScores(); - vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer( pds[i] ); - for (size_t j = 0; j<scores.size(); ++j){ - - if (labeledOutput && (i == 0) ){ - if ((j == 0) || (j == pd_numinputscore)){ - lastName = pds[i]->GetScoreProducerWeightShortName(j); - out << " " << lastName << ":"; - } - } - out << " " << scores[j]; - } + size_t pd_numinputscore = pds[i]->GetNumInputScores(); + vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer( pds[i] ); + for (size_t j = 0; j<scores.size(); ++j){ + if (labeledOutput && (i == 0) ){ + if ((j == 0) || (j == pd_numinputscore)){ + lastName = pds[i]->GetScoreProducerWeightShortName(j); + out << " " << lastName << ":"; + } + } + out << " " << scores[j]; + } } } @@ -448,26 +455,33 @@ void IOWrapper::OutputNBestList(const ChartTrellisPathList &nBestList, const Cha // generation const vector<GenerationDictionary*>& gds = system->GetGenerationDictionaries(); if (gds.size() > 0) { - for( size_t i=0; i<gds.size(); i++ ) { - size_t pd_numinputscore = gds[i]->GetNumInputScores(); - vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer( gds[i] ); - for (size_t j = 0; j<scores.size(); ++j){ - - if (labeledOutput && (i == 0) ){ - if ((j == 0) || (j == pd_numinputscore)){ - lastName = gds[i]->GetScoreProducerWeightShortName(j); - out << " " << lastName << ":"; - } - } - out << " " << scores[j]; - } + size_t pd_numinputscore = gds[i]->GetNumInputScores(); + vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer( gds[i] ); + for (size_t j = 0; j<scores.size(); ++j){ + if (labeledOutput && (i == 0) ){ + if ((j == 0) || (j == pd_numinputscore)){ + lastName = gds[i]->GetScoreProducerWeightShortName(j); + out << " " << lastName << ":"; + } + } + out << " " << scores[j]; + } } } + // output stateless sparse features + lastName = ""; + + const vector<const StatelessFeatureFunction*>& slf = system->GetStatelessFeatureFunctions(); + for( size_t i=0; i<slf.size(); i++ ) { + if (slf[i]->GetNumScoreComponents() == ScoreProducer::unlimited) { + OutputSparseFeatureScores( out, path, slf[i], lastName ); + } + } // total - out << " |||" << path.GetTotalScore(); + out << " ||| " << path.GetTotalScore(); /* if (includeAlignment) { @@ -498,6 +512,32 @@ void IOWrapper::OutputNBestList(const ChartTrellisPathList &nBestList, const Cha m_nBestOutputCollector->Write(translationId, out.str()); } +void IOWrapper::OutputSparseFeatureScores( std::ostream& out, const ChartTrellisPath &path, const FeatureFunction *ff, std::string &lastName ) +{ + const StaticData &staticData = StaticData::Instance(); + bool labeledOutput = staticData.IsLabeledNBestList(); + const FVector scores = path.GetScoreBreakdown().GetVectorForProducer( ff ); + + // report weighted aggregate + if (! ff->GetSparseFeatureReporting()) { + const FVector &weights = staticData.GetAllWeights().GetScoresVector(); + if (labeledOutput && !boost::contains(ff->GetScoreProducerDescription(), ":")) + out << " " << ff->GetScoreProducerWeightShortName() << ":"; + out << " " << scores.inner_product(weights); + } + + // report each feature + else { + for(FVector::FNVmap::const_iterator i = scores.cbegin(); i != scores.cend(); i++) { + if (i->second != 0) { // do not report zero-valued features + if (labeledOutput) + out << " " << i->first << ":"; + out << " " << i->second; + } + } + } +} + void IOWrapper::FixPrecision(std::ostream &stream, size_t size) { stream.setf(std::ios::fixed); diff --git a/moses-chart-cmd/src/IOWrapper.h b/moses-chart-cmd/src/IOWrapper.h index 35bfa07ab..2a845f29d 100644 --- a/moses-chart-cmd/src/IOWrapper.h +++ b/moses-chart-cmd/src/IOWrapper.h @@ -44,6 +44,8 @@ POSSIBILITY OF SUCH DAMAGE. #include "OutputCollector.h" #include "ChartHypothesis.h" +#include "ChartTrellisPath.h" + namespace Moses { class FactorCollection; @@ -86,6 +88,7 @@ public: void OutputBestHypo(const Moses::ChartHypothesis *hypo, long translationId, bool reportSegmentation, bool reportAllFactors); void OutputBestHypo(const std::vector<const Moses::Factor*>& mbrBestHypo, long translationId, bool reportSegmentation, bool reportAllFactors); void OutputNBestList(const Moses::ChartTrellisPathList &nBestList, const Moses::ChartHypothesis *bestHypo, const Moses::TranslationSystem* system, long translationId); + void OutputSparseFeatureScores(std::ostream& out, const Moses::ChartTrellisPath &path, const Moses::FeatureFunction *ff, std::string &lastName); void OutputDetailedTranslationReport(const Moses::ChartHypothesis *hypo, const Moses::Sentence &sentence, long translationId); void Backtrack(const Moses::ChartHypothesis *hypo); diff --git a/moses-chart-cmd/src/Main.cpp b/moses-chart-cmd/src/Main.cpp index 7b2dc79c0..607a6eb00 100644 --- a/moses-chart-cmd/src/Main.cpp +++ b/moses-chart-cmd/src/Main.cpp @@ -86,6 +86,7 @@ public: VERBOSE(2,"\nTRANSLATING(" << lineNumber << "): " << *m_source); + if ((*m_source).GetSize() == 0) return; ChartManager manager(*m_source, &system); manager.ProcessSentence(); @@ -161,19 +162,26 @@ bool ReadInput(IOWrapper &ioWrapper, InputTypeEnum inputType, InputType*& source } return (source ? true : false); } - static void PrintFeatureWeight(const FeatureFunction* ff) { - - size_t weightStart = StaticData::Instance().GetScoreIndexManager().GetBeginIndex(ff->GetScoreBookkeepingID()); - size_t weightEnd = StaticData::Instance().GetScoreIndexManager().GetEndIndex(ff->GetScoreBookkeepingID()); - for (size_t i = weightStart; i < weightEnd; ++i) { - cout << ff->GetScoreProducerDescription(i-weightStart) << " " << ff->GetScoreProducerWeightShortName(i-weightStart) << " " - << StaticData::Instance().GetAllWeights()[i] << endl; + size_t numScoreComps = ff->GetNumScoreComponents(); + if (numScoreComps != ScoreProducer::unlimited) { + vector<float> values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff); + for (size_t i = 0; i < numScoreComps; ++i) + cout << ff->GetScoreProducerDescription() << " " + << ff->GetScoreProducerWeightShortName() << " " + << values[i] << endl; + } + else { + if (ff->GetSparseProducerWeight() == 1) + cout << ff->GetScoreProducerDescription() << " " << + ff->GetScoreProducerWeightShortName() << " sparse" << endl; + else + cout << ff->GetScoreProducerDescription() << " " << + ff->GetScoreProducerWeightShortName() << " " << ff->GetSparseProducerWeight() << endl; } } - static void ShowWeights() { cout.precision(6); @@ -181,19 +189,13 @@ static void ShowWeights() const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT); const vector<const StatelessFeatureFunction*>& slf =system.GetStatelessFeatureFunctions(); const vector<const StatefulFeatureFunction*>& sff = system.GetStatefulFeatureFunctions(); - const vector<PhraseDictionaryFeature*>& pds = system.GetPhraseDictionaries(); - const vector<GenerationDictionary*>& gds = system.GetGenerationDictionaries(); for (size_t i = 0; i < sff.size(); ++i) { PrintFeatureWeight(sff[i]); } - for (size_t i = 0; i < pds.size(); ++i) { - PrintFeatureWeight(pds[i]); - } - for (size_t i = 0; i < gds.size(); ++i) { - PrintFeatureWeight(gds[i]); - } for (size_t i = 0; i < slf.size(); ++i) { - PrintFeatureWeight(slf[i]); + if (slf[i]->GetScoreProducerWeightShortName() != "u") { + PrintFeatureWeight(slf[i]); + } } } @@ -206,49 +208,41 @@ int main(int argc, char* argv[]) for(int i=0; i<argc; ++i) TRACE_ERR(argv[i]<<" "); TRACE_ERR(endl); } - + IOWrapper::FixPrecision(cout); IOWrapper::FixPrecision(cerr); - + // load data structures Parameter parameter; if (!parameter.LoadParam(argc, argv)) { return EXIT_FAILURE; } - + const StaticData &staticData = StaticData::Instance(); if (!StaticData::LoadDataStatic(¶meter, argv[0])) return EXIT_FAILURE; - + if (parameter.isParamSpecified("show-weights")) { ShowWeights(); exit(0); } - + CHECK(staticData.GetSearchAlgorithm() == ChartDecoding); - + // set up read/writing class IOWrapper *ioWrapper = GetIOWrapper(staticData); // check on weights - vector<float> weights = staticData.GetAllWeights(); + const ScoreComponentCollection& weights = staticData.GetAllWeights(); IFVERBOSE(2) { - TRACE_ERR("The score component vector looks like this:\n" << staticData.GetScoreIndexManager()); - TRACE_ERR("The global weight vector looks like this:"); - for (size_t j=0; j<weights.size(); j++) { - TRACE_ERR(" " << weights[j]); - } + TRACE_ERR("The global weight vector looks like this: "); + TRACE_ERR(weights); TRACE_ERR("\n"); } - // every score must have a weight! check that here: - if(weights.size() != staticData.GetScoreIndexManager().GetTotalNumberOfScores()) { - TRACE_ERR("ERROR: " << staticData.GetScoreIndexManager().GetTotalNumberOfScores() << " score components, but " << weights.size() << " weights defined" << std::endl); - return EXIT_FAILURE; - } - + if (ioWrapper == NULL) return EXIT_FAILURE; - + #ifdef WITH_THREADS ThreadPool pool(staticData.ThreadCount()); #endif diff --git a/moses-cmd/src/IOWrapper.cpp b/moses-cmd/src/IOWrapper.cpp index 053735c96..80958ace0 100644 --- a/moses-cmd/src/IOWrapper.cpp +++ b/moses-cmd/src/IOWrapper.cpp @@ -42,7 +42,9 @@ POSSIBILITY OF SUCH DAMAGE. #include "TrellisPathList.h" #include "StaticData.h" #include "DummyScoreProducers.h" +#include "FeatureVector.h" #include "InputFileStream.h" +#include <boost/algorithm/string.hpp> using namespace std; using namespace Moses; @@ -382,30 +384,9 @@ void OutputNBest(std::ostream& out, const Moses::TrellisPathList &nBestList, con } out << " |||"; - std::string lastName = ""; - const vector<const StatefulFeatureFunction*>& sff = system->GetStatefulFeatureFunctions(); - for( size_t i=0; i<sff.size(); i++ ) { - if( labeledOutput && lastName != sff[i]->GetScoreProducerWeightShortName() ) { - lastName = sff[i]->GetScoreProducerWeightShortName(); - out << " " << lastName << ":"; - } - vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer( sff[i] ); - for (size_t j = 0; j<scores.size(); ++j) { - out << " " << scores[j]; - } - } - - const vector<const StatelessFeatureFunction*>& slf = system->GetStatelessFeatureFunctions(); - for( size_t i=0; i<slf.size(); i++ ) { - if( labeledOutput && lastName != slf[i]->GetScoreProducerWeightShortName() ) { - lastName = slf[i]->GetScoreProducerWeightShortName(); - out << " " << lastName << ":"; - } - vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer( slf[i] ); - for (size_t j = 0; j<scores.size(); ++j) { - out << " " << scores[j]; - } - } + // print scores with feature names + OutputAllFeatureScores( out, system, path ); + string lastName; // translation components const vector<PhraseDictionaryFeature*>& pds = system->GetPhraseDictionaries(); @@ -491,8 +472,66 @@ void OutputNBest(std::ostream& out, const Moses::TrellisPathList &nBestList, con out << endl; } + out << std::flush; +} + +void OutputAllFeatureScores( std::ostream& out, const TranslationSystem* system, const TrellisPath &path ) +{ + std::string lastName = ""; + const vector<const StatefulFeatureFunction*>& sff = system->GetStatefulFeatureFunctions(); + for( size_t i=0; i<sff.size(); i++ ) + if (sff[i]->GetScoreProducerWeightShortName() != "bl") + OutputFeatureScores( out, path, sff[i], lastName ); + + const vector<const StatelessFeatureFunction*>& slf = system->GetStatelessFeatureFunctions(); + for( size_t i=0; i<slf.size(); i++ ) + if (slf[i]->GetScoreProducerWeightShortName() != "u" && + slf[i]->GetScoreProducerWeightShortName() != "tm" && + slf[i]->GetScoreProducerWeightShortName() != "I" && + slf[i]->GetScoreProducerWeightShortName() != "g") + OutputFeatureScores( out, path, slf[i], lastName ); +} + +void OutputFeatureScores( std::ostream& out, const TrellisPath &path, const FeatureFunction *ff, std::string &lastName ) +{ + const StaticData &staticData = StaticData::Instance(); + bool labeledOutput = staticData.IsLabeledNBestList(); + + // regular features (not sparse) + if (ff->GetNumScoreComponents() != ScoreProducer::unlimited) { + if( labeledOutput && lastName != ff->GetScoreProducerWeightShortName() ) { + lastName = ff->GetScoreProducerWeightShortName(); + out << " " << lastName << ":"; + } + vector<float> scores = path.GetScoreBreakdown().GetScoresForProducer( ff ); + for (size_t j = 0; j<scores.size(); ++j) { + out << " " << scores[j]; + } + } + + // sparse features + else { + const FVector scores = path.GetScoreBreakdown().GetVectorForProducer( ff ); + + // report weighted aggregate + if (! ff->GetSparseFeatureReporting()) { + const FVector &weights = staticData.GetAllWeights().GetScoresVector(); + if (labeledOutput && !boost::contains(ff->GetScoreProducerDescription(), ":")) + out << " " << ff->GetScoreProducerWeightShortName() << ":"; + out << " " << scores.inner_product(weights); + } - out <<std::flush; + // report each feature + else { + for(FVector::FNVmap::const_iterator i = scores.cbegin(); i != scores.cend(); i++) + out << " " << i->first << ": " << i->second; + /* if (i->second != 0) { // do not report zero-valued features + float weight = staticData.GetSparseWeight(i->first); + if (weight != 0) + out << " " << i->first << "=" << weight; + }*/ + } + } } void OutputLatticeMBRNBest(std::ostream& out, const vector<LatticeMBRSolution>& solutions,long translationId) diff --git a/moses-cmd/src/IOWrapper.h b/moses-cmd/src/IOWrapper.h index e44208002..fc811f248 100644 --- a/moses-cmd/src/IOWrapper.h +++ b/moses-cmd/src/IOWrapper.h @@ -128,6 +128,8 @@ bool ReadInput(IOWrapper &ioWrapper, Moses::InputTypeEnum inputType, Moses::Inpu void OutputBestSurface(std::ostream &out, const Moses::Hypothesis *hypo, const std::vector<Moses::FactorType> &outputFactorOrder, bool reportSegmentation, bool reportAllFactors); void OutputNBest(std::ostream& out, const Moses::TrellisPathList &nBestList, const std::vector<Moses::FactorType>&, const Moses::TranslationSystem* system, long translationId, bool reportSegmentation); +void OutputAllFeatureScores(std::ostream& out, const Moses::TranslationSystem* system, const Moses::TrellisPath &path); +void OutputFeatureScores(std::ostream& out, const Moses::TrellisPath &path, const Moses::FeatureFunction *ff, std::string &lastName); void OutputLatticeMBRNBest(std::ostream& out, const std::vector<LatticeMBRSolution>& solutions,long translationId); void OutputBestHypo(const std::vector<Moses::Word>& mbrBestHypo, long /*translationId*/, bool reportSegmentation, bool reportAllFactors, std::ostream& out); diff --git a/moses-cmd/src/Jamfile b/moses-cmd/src/Jamfile index c4137daff..8f9296f7f 100644 --- a/moses-cmd/src/Jamfile +++ b/moses-cmd/src/Jamfile @@ -1,7 +1,10 @@ alias deps : ../../moses/src//moses ; -exe moses : Main.cpp mbr.cpp IOWrapper.cpp TranslationAnalysis.cpp LatticeMBR.cpp deps ; -exe lmbrgrid : LatticeMBRGrid.cpp LatticeMBR.cpp IOWrapper.cpp deps ; +exe moses : Main.cpp mbr.cpp TranslationAnalysis.cpp LatticeMBR.cpp IOWrapper_lib ; +exe lmbrgrid : LatticeMBRGrid.cpp LatticeMBR.cpp IOWrapper_lib ; alias programs : moses lmbrgrid ; +install legacy-install : programs : <location>. <install-type>EXE <install-dependencies>on <link>shared:<dll-path>$(TOP)/moses-cmd/src <link>shared:<install-type>LIB ; + +lib IOWrapper_lib : IOWrapper.cpp deps : : : <include>. ; diff --git a/moses-cmd/src/Main.cpp b/moses-cmd/src/Main.cpp index f73da1665..891be4167 100644 --- a/moses-cmd/src/Main.cpp +++ b/moses-cmd/src/Main.cpp @@ -113,6 +113,7 @@ public: // execute the translation // note: this executes the search, resulting in a search graph // we still need to apply the decision rule (MAP, MBR, ...) + if ((*m_source).GetSize() == 0) return; Manager manager(m_lineNumber, *m_source,staticData.GetSearchAlgorithm(), &system); manager.ProcessSentence(); @@ -314,16 +315,24 @@ private: static void PrintFeatureWeight(const FeatureFunction* ff) { - - size_t weightStart = StaticData::Instance().GetScoreIndexManager().GetBeginIndex(ff->GetScoreBookkeepingID()); - size_t weightEnd = StaticData::Instance().GetScoreIndexManager().GetEndIndex(ff->GetScoreBookkeepingID()); - for (size_t i = weightStart; i < weightEnd; ++i) { - cout << ff->GetScoreProducerDescription(i-weightStart) << " " << ff->GetScoreProducerWeightShortName(i-weightStart) << " " - << StaticData::Instance().GetAllWeights()[i] << endl; + size_t numScoreComps = ff->GetNumScoreComponents(); + if (numScoreComps != ScoreProducer::unlimited) { + vector<float> values = StaticData::Instance().GetAllWeights().GetScoresForProducer(ff); + for (size_t i = 0; i < numScoreComps; ++i) + cout << ff->GetScoreProducerDescription() << " " + << ff->GetScoreProducerWeightShortName() << " " + << values[i] << endl; + } + else { + if (ff->GetSparseProducerWeight() == 1) + cout << ff->GetScoreProducerDescription() << " " << + ff->GetScoreProducerWeightShortName() << " sparse" << endl; + else + cout << ff->GetScoreProducerDescription() << " " << + ff->GetScoreProducerWeightShortName() << " " << ff->GetSparseProducerWeight() << endl; } } - static void ShowWeights() { fix(cout,6); @@ -331,19 +340,13 @@ static void ShowWeights() const TranslationSystem& system = staticData.GetTranslationSystem(TranslationSystem::DEFAULT); const vector<const StatelessFeatureFunction*>& slf =system.GetStatelessFeatureFunctions(); const vector<const StatefulFeatureFunction*>& sff = system.GetStatefulFeatureFunctions(); - const vector<PhraseDictionaryFeature*>& pds = system.GetPhraseDictionaries(); - const vector<GenerationDictionary*>& gds = system.GetGenerationDictionaries(); for (size_t i = 0; i < sff.size(); ++i) { PrintFeatureWeight(sff[i]); } for (size_t i = 0; i < slf.size(); ++i) { - PrintFeatureWeight(slf[i]); - } - for (size_t i = 0; i < pds.size(); ++i) { - PrintFeatureWeight(pds[i]); - } - for (size_t i = 0; i < gds.size(); ++i) { - PrintFeatureWeight(gds[i]); + if (slf[i]->GetScoreProducerWeightShortName() != "u") { + PrintFeatureWeight(slf[i]); + } } } @@ -357,68 +360,60 @@ int main(int argc, char** argv) #ifdef HAVE_PROTOBUF GOOGLE_PROTOBUF_VERIFY_VERSION; #endif - + // echo command line, if verbose IFVERBOSE(1) { TRACE_ERR("command: "); for(int i=0; i<argc; ++i) TRACE_ERR(argv[i]<<" "); TRACE_ERR(endl); } - + // set number of significant decimals in output fix(cout,PRECISION); fix(cerr,PRECISION); - + // load all the settings into the Parameter class // (stores them as strings, or array of strings) Parameter* params = new Parameter(); if (!params->LoadParam(argc,argv)) { exit(1); } - - + + // initialize all "global" variables, which are stored in StaticData // note: this also loads models such as the language model, etc. if (!StaticData::LoadDataStatic(params, argv[0])) { exit(1); } - + // setting "-show-weights" -> just dump out weights and exit if (params->isParamSpecified("show-weights")) { ShowWeights(); exit(0); } - + // shorthand for accessing information in StaticData const StaticData& staticData = StaticData::Instance(); - - + + //initialise random numbers srand(time(NULL)); - + // set up read/writing class IOWrapper* ioWrapper = GetIOWrapper(staticData); if (!ioWrapper) { cerr << "Error; Failed to create IO object" << endl; exit(1); } - + // check on weights - vector<float> weights = staticData.GetAllWeights(); + const ScoreComponentCollection& weights = staticData.GetAllWeights(); IFVERBOSE(2) { - TRACE_ERR("The score component vector looks like this:\n" << staticData.GetScoreIndexManager()); - TRACE_ERR("The global weight vector looks like this:"); - for (size_t j=0; j<weights.size(); j++) { - TRACE_ERR(" " << weights[j]); - } + TRACE_ERR("The global weight vector looks like this: "); + TRACE_ERR(weights); TRACE_ERR("\n"); } - // every score must have a weight! check that here: - if(weights.size() != staticData.GetScoreIndexManager().GetTotalNumberOfScores()) { - TRACE_ERR("ERROR: " << staticData.GetScoreIndexManager().GetTotalNumberOfScores() << " score components, but " << weights.size() << " weights defined" << std::endl); - exit(1); - } - + // initialize output streams // note: we can't just write to STDOUT or files // because multithreading may return sentences in shuffled order diff --git a/moses-cmd/src/TranslationAnalysis.cpp b/moses-cmd/src/TranslationAnalysis.cpp index 89da48301..a7795a27b 100644 --- a/moses-cmd/src/TranslationAnalysis.cpp +++ b/moses-cmd/src/TranslationAnalysis.cpp @@ -118,9 +118,10 @@ void PrintTranslationAnalysis(const TranslationSystem* system, std::ostream &os, os << "\tdropped=" << *dwi << std::endl; } } - os << std::endl << "SCORES (UNWEIGHTED/WEIGHTED): "; - StaticData::Instance().GetScoreIndexManager().PrintLabeledWeightedScores(os, translationPath.back()->GetScoreBreakdown(), StaticData::Instance().GetAllWeights()); - os << std::endl; + os << std::endl << "SCORES (UNWEIGHTED/WEIGHTED): "; + os << translationPath.back()->GetScoreBreakdown(); + os << " weighted(TODO)"; + os << std::endl; } } diff --git a/moses/src/AlignmentInfo.h b/moses/src/AlignmentInfo.h index 179903148..01d30013d 100644 --- a/moses/src/AlignmentInfo.h +++ b/moses/src/AlignmentInfo.h @@ -19,9 +19,11 @@ #pragma once +#include <iostream> #include <ostream> #include <set> #include <vector> +#include <cstdlib> namespace Moses { @@ -33,14 +35,14 @@ class AlignmentInfoCollection; */ class AlignmentInfo { - typedef std::set<std::pair<size_t,size_t> > CollType; - friend std::ostream& operator<<(std::ostream &, const AlignmentInfo &); friend struct AlignmentInfoOrderer; friend class AlignmentInfoCollection; public: + typedef std::set<std::pair<size_t,size_t> > CollType; typedef std::vector<size_t> NonTermIndexMap; + typedef std::vector<size_t> TermIndexMap; typedef CollType::const_iterator const_iterator; const_iterator begin() const { return m_collection.begin(); } @@ -53,7 +55,17 @@ class AlignmentInfo const NonTermIndexMap &GetNonTermIndexMap() const { return m_nonTermIndexMap; } - + + // only used for hierarchical models, contains terminal alignments + const CollType &GetTerminalAlignments() const { + return m_terminalCollection; + } + + // for phrase-based models, this contains all alignments, for hierarchical models only the NT alignments + const CollType &GetAlignments() const { + return m_collection; + } + size_t GetSize() const { return m_collection.size(); } std::vector< const std::pair<size_t,size_t>* > GetSortedAlignments() const; @@ -65,10 +77,29 @@ class AlignmentInfo { BuildNonTermIndexMap(); } - + + // use this for hierarchical models + explicit AlignmentInfo(const std::set<std::pair<size_t,size_t> > &pairs, int* indicator) + { + // split alignment set in terminals and non-terminals + std::set<std::pair<size_t,size_t> > terminalSet; + std::set<std::pair<size_t,size_t> > nonTerminalSet; + std::set<std::pair<size_t,size_t> >::iterator iter; + for (iter = pairs.begin(); iter != pairs.end(); ++iter) { + if (*indicator == 1) nonTerminalSet.insert(*iter); + else terminalSet.insert(*iter); + indicator++; + } + m_collection = nonTerminalSet; + m_terminalCollection = terminalSet; + + BuildNonTermIndexMap(); + } + void BuildNonTermIndexMap(); CollType m_collection; + CollType m_terminalCollection; NonTermIndexMap m_nonTermIndexMap; }; @@ -78,6 +109,8 @@ class AlignmentInfo struct AlignmentInfoOrderer { bool operator()(const AlignmentInfo &a, const AlignmentInfo &b) const { + if (a.m_collection == b.m_collection) + return a.m_terminalCollection < b.m_terminalCollection; return a.m_collection < b.m_collection; } }; diff --git a/moses/src/AlignmentInfoCollection.cpp b/moses/src/AlignmentInfoCollection.cpp index 094811daa..4569b374b 100644 --- a/moses/src/AlignmentInfoCollection.cpp +++ b/moses/src/AlignmentInfoCollection.cpp @@ -53,4 +53,12 @@ const AlignmentInfo *AlignmentInfoCollection::Add( return &(*ret.first); } +const AlignmentInfo *AlignmentInfoCollection::Add( + const std::set<std::pair<size_t,size_t> > &pairs, int* indicator) +{ + std::pair<AlignmentInfoSet::iterator, bool> ret = + m_collection.insert(AlignmentInfo(pairs, indicator)); + return &(*ret.first); +} + } diff --git a/moses/src/AlignmentInfoCollection.h b/moses/src/AlignmentInfoCollection.h index 9c7f75e13..708ba285c 100644 --- a/moses/src/AlignmentInfoCollection.h +++ b/moses/src/AlignmentInfoCollection.h @@ -46,6 +46,7 @@ class AlignmentInfoCollection * one is inserted. */ const AlignmentInfo *Add(const std::set<std::pair<size_t,size_t> > &); + const AlignmentInfo *Add(const std::set<std::pair<size_t,size_t> > &, int* indicator); //! Returns a pointer to an empty AlignmentInfo object. const AlignmentInfo &GetEmptyAlignmentInfo() const; diff --git a/moses/src/BilingualDynSuffixArray.cpp b/moses/src/BilingualDynSuffixArray.cpp index c92da0017..1e1ef649e 100644 --- a/moses/src/BilingualDynSuffixArray.cpp +++ b/moses/src/BilingualDynSuffixArray.cpp @@ -395,7 +395,7 @@ SAPhrase BilingualDynSuffixArray::TrgPhraseFromSntIdx(const PhrasePair& phrasepa return phraseIds; } -TargetPhrase* BilingualDynSuffixArray::GetMosesFactorIDs(const SAPhrase& phrase) const +TargetPhrase* BilingualDynSuffixArray::GetMosesFactorIDs(const SAPhrase& phrase, const Phrase& sourcePhrase) const { TargetPhrase* targetPhrase = new TargetPhrase(Output); for(size_t i=0; i < phrase.words.size(); ++i) { // look up trg words @@ -403,6 +403,7 @@ TargetPhrase* BilingualDynSuffixArray::GetMosesFactorIDs(const SAPhrase& phrase) CHECK(word != m_trgVocab->GetkOOVWord()); targetPhrase->AddWord(word); } + targetPhrase->SetSourcePhrase(sourcePhrase); // scoring return targetPhrase; } @@ -464,7 +465,7 @@ void BilingualDynSuffixArray::GetTargetPhrasesByLexicalWeight(const Phrase& src, std::multimap<Scores, const SAPhrase*, ScoresComp>::reverse_iterator ritr; for(ritr = phraseScores.rbegin(); ritr != phraseScores.rend(); ++ritr) { Scores scoreVector = ritr->first; - TargetPhrase *targetPhrase = GetMosesFactorIDs(*ritr->second); + TargetPhrase *targetPhrase = GetMosesFactorIDs(*ritr->second, src); target.push_back(make_pair( scoreVector, targetPhrase)); if(target.size() == m_maxSampleSize) break; } diff --git a/moses/src/BilingualDynSuffixArray.h b/moses/src/BilingualDynSuffixArray.h index c614f1738..18c42f342 100644 --- a/moses/src/BilingualDynSuffixArray.h +++ b/moses/src/BilingualDynSuffixArray.h @@ -132,7 +132,7 @@ private: int SampleSelection(std::vector<unsigned>&, int = 300) const; std::vector<int> GetSntIndexes(std::vector<unsigned>&, int, const std::vector<unsigned>&) const; - TargetPhrase* GetMosesFactorIDs(const SAPhrase&) const; + TargetPhrase* GetMosesFactorIDs(const SAPhrase&, const Phrase& sourcePhrase) const; SAPhrase TrgPhraseFromSntIdx(const PhrasePair&) const; bool GetLocalVocabIDs(const Phrase&, SAPhrase &) const; void CacheWordProbs(wordID_t) const; diff --git a/moses/src/BleuScoreFeature.cpp b/moses/src/BleuScoreFeature.cpp new file mode 100644 index 000000000..b51f1ebdb --- /dev/null +++ b/moses/src/BleuScoreFeature.cpp @@ -0,0 +1,801 @@ +#include "BleuScoreFeature.h" + +#include "StaticData.h" + +using namespace std; + +namespace Moses { + +size_t BleuScoreState::bleu_order = 4; + +BleuScoreState::BleuScoreState(): m_words(1), + m_source_length(0), + m_target_length(0), + m_scaled_ref_length(0), + m_ngram_counts(bleu_order), + m_ngram_matches(bleu_order) +{ +} + +int BleuScoreState::Compare(const FFState& o) const +{ + if (&o == this) + return 0; + + const StaticData &staticData = StaticData::Instance(); + SearchAlgorithm searchAlgorithm = staticData.GetSearchAlgorithm(); + bool chartDecoding = (searchAlgorithm == ChartDecoding); + if (chartDecoding) + return 0; + + const BleuScoreState& other = dynamic_cast<const BleuScoreState&>(o); + int c = m_words.Compare(other.m_words); + if (c != 0) + return c; + + /*for(size_t i = 0; i < m_ngram_counts.size(); i++) { + if (m_ngram_counts[i] < other.m_ngram_counts[i]) + return -1; + if (m_ngram_counts[i] > other.m_ngram_counts[i]) + return 1; + if (m_ngram_matches[i] < other.m_ngram_matches[i]) + return -1; + if (m_ngram_matches[i] > other.m_ngram_matches[i]) + return 1; + }*/ + + return 0; +} +std::ostream& operator<<(std::ostream& out, const BleuScoreState& state) { + state.print(out); + return out; +} + +void BleuScoreState::print(std::ostream& out) const { + out << "ref=" << m_scaled_ref_length + << ";source=" << m_source_length + << ";target=" << m_target_length << ";counts="; + for (size_t i = 0; i < bleu_order; ++i) { + out << m_ngram_matches[i] << "/" << m_ngram_counts[i] << ","; + } + out << "ctxt=" << m_words; + +} + +void BleuScoreState::AddNgramCountAndMatches(std::vector< size_t >& counts, + std::vector< size_t >& matches) { + for (size_t order = 0; order < BleuScoreState::bleu_order; ++order) { + m_ngram_counts[order] += counts[order]; + m_ngram_matches[order] += matches[order]; + } +} + +void BleuScoreFeature::PrintHistory(std::ostream& out) const { + out << "source length history=" << m_source_length_history << endl; + out << "target length history=" << m_target_length_history << endl; + out << "ref length history=" << m_ref_length_history << endl; + + for (size_t i = 0; i < BleuScoreState::bleu_order; ++i) { + out << "match history/count history (" << i << "):" << m_match_history[i] << "/" << m_count_history[i] << endl; + } +} + +void BleuScoreFeature::SetBleuParameters(bool disable, bool sentenceBleu, bool scaleByInputLength, bool scaleByAvgInputLength, + bool scaleByInverseLength, bool scaleByAvgInverseLength, + float scaleByX, float historySmoothing, size_t scheme, bool simpleHistoryBleu) { + m_enabled = !disable; + m_sentence_bleu = sentenceBleu; + m_simple_history_bleu = simpleHistoryBleu; + m_scale_by_input_length = scaleByInputLength; + m_scale_by_avg_input_length = scaleByAvgInputLength; + m_scale_by_inverse_length = scaleByInverseLength; + m_scale_by_avg_inverse_length = scaleByAvgInverseLength; + m_scale_by_x = scaleByX; + m_historySmoothing = historySmoothing; + m_smoothing_scheme = (SmoothingScheme)scheme; +} + +// Incoming references (refs) are stored as refs[file_id][[sent_id][reference]] +// This data structure: m_refs[sent_id][[vector<length>][ngrams]] +void BleuScoreFeature::LoadReferences(const std::vector< std::vector< std::string > >& refs) +{ + m_refs.clear(); + FactorCollection& fc = FactorCollection::Instance(); + for (size_t file_id = 0; file_id < refs.size(); file_id++) { + for (size_t sent_id = 0; sent_id < refs[file_id].size(); sent_id++) { + const string& ref = refs[file_id][sent_id]; + vector<string> refTokens = Tokenize(ref); + if (file_id == 0) + m_refs[sent_id] = pair<vector<size_t>,NGrams>(); + pair<vector<size_t>,NGrams>& ref_pair = m_refs[sent_id]; + (ref_pair.first).push_back(refTokens.size()); + for (size_t order = 1; order <= BleuScoreState::bleu_order; order++) { + for (size_t end_idx = order; end_idx <= refTokens.size(); end_idx++) { + Phrase ngram(1); + for (size_t s_idx = end_idx - order; s_idx < end_idx; s_idx++) { + const Factor* f = fc.AddFactor(Output, 0, refTokens[s_idx]); + Word w; + w.SetFactor(0, f); + ngram.AddWord(w); + } + ref_pair.second[ngram] += 1; + } + } + } + } + +// cerr << "Number of ref files: " << refs.size() << endl; +// for (size_t i = 0; i < m_refs.size(); ++i) { +// cerr << "Sent id " << i << ", number of references: " << (m_refs[i].first).size() << endl; +// } +} + +void BleuScoreFeature::SetCurrSourceLength(size_t source_length) { + m_cur_source_length = source_length; +} +void BleuScoreFeature::SetCurrNormSourceLength(size_t source_length) { + m_cur_norm_source_length = source_length; +} + +// m_refs[sent_id][[vector<length>][ngrams]] +void BleuScoreFeature::SetCurrShortestRefLength(size_t sent_id) { + // look for shortest reference + int shortestRef = -1; + for (size_t i = 0; i < (m_refs[sent_id].first).size(); ++i) { + if (shortestRef == -1 || (m_refs[sent_id].first)[i] < shortestRef) + shortestRef = (m_refs[sent_id].first)[i]; + } + m_cur_ref_length = shortestRef; +// cerr << "Set shortest cur_ref_length: " << m_cur_ref_length << endl; +} + +void BleuScoreFeature::SetCurrAvgRefLength(size_t sent_id) { + // compute average reference length + size_t sum = 0; + size_t numberRefs = (m_refs[sent_id].first).size(); + for (size_t i = 0; i < numberRefs; ++i) { + sum += (m_refs[sent_id].first)[i]; + } + m_cur_ref_length = (float)sum/numberRefs; +// cerr << "Set average cur_ref_length: " << m_cur_ref_length << endl; +} + +void BleuScoreFeature::SetCurrReferenceNgrams(size_t sent_id) { + m_cur_ref_ngrams = m_refs[sent_id].second; +} + +size_t BleuScoreFeature::GetShortestRefIndex(size_t ref_id) { + // look for shortest reference + int shortestRef = -1; + size_t shortestRefIndex = 0; + for (size_t i = 0; i < (m_refs[ref_id].first).size(); ++i) { + if (shortestRef == -1 || (m_refs[ref_id].first)[i] < shortestRef) { + shortestRef = (m_refs[ref_id].first)[i]; + shortestRefIndex = i; + } + } + return shortestRefIndex; +} + +/* + * Update the pseudo-document O after each translation of a source sentence. + * (O is an exponentially-weighted moving average of vectors c(e;{r_k})) + * O = m_historySmoothing * (O + c(e_oracle)) + * O_f = m_historySmoothing * (O_f + |f|) input length of pseudo-document + */ +void BleuScoreFeature::UpdateHistory(const vector< const Word* >& hypo) { + Phrase phrase(hypo); + std::vector< size_t > ngram_counts(BleuScoreState::bleu_order); + std::vector< size_t > ngram_matches(BleuScoreState::bleu_order); + + // compute vector c(e;{r_k}): + // vector of effective reference length, number of ngrams in e, number of ngram matches between e and r_k + GetNgramMatchCounts(phrase, m_cur_ref_ngrams, ngram_counts, ngram_matches, 0); + + // update counts and matches for every ngram length with counts from hypo + for (size_t i = 0; i < BleuScoreState::bleu_order; i++) { + m_count_history[i] = m_historySmoothing * (m_count_history[i] + ngram_counts[i]); + m_match_history[i] = m_historySmoothing * (m_match_history[i] + ngram_matches[i]); + } + + // update counts for reference and target length + m_source_length_history = m_historySmoothing * (m_source_length_history + m_cur_source_length); + m_target_length_history = m_historySmoothing * (m_target_length_history + hypo.size()); + m_ref_length_history = m_historySmoothing * (m_ref_length_history + m_cur_ref_length); +} + +/* + * Update history with a batch of translations + */ +void BleuScoreFeature::UpdateHistory(const vector< vector< const Word* > >& hypos, vector<size_t>& sourceLengths, vector<size_t>& ref_ids, size_t rank, size_t epoch) { + for (size_t ref_id = 0; ref_id < hypos.size(); ++ref_id){ + Phrase phrase(hypos[ref_id]); + std::vector< size_t > ngram_counts(BleuScoreState::bleu_order); + std::vector< size_t > ngram_matches(BleuScoreState::bleu_order); + + // set current source and reference information for each oracle in the batch + size_t cur_source_length = sourceLengths[ref_id]; + size_t hypo_length = hypos[ref_id].size(); + size_t cur_ref_length = GetClosestRefLength(ref_ids[ref_id], hypo_length); + NGrams cur_ref_ngrams = m_refs[ref_ids[ref_id]].second; + cerr << "reference length: " << cur_ref_length << endl; + + // compute vector c(e;{r_k}): + // vector of effective reference length, number of ngrams in e, number of ngram matches between e and r_k + GetNgramMatchCounts(phrase, cur_ref_ngrams, ngram_counts, ngram_matches, 0); + + // update counts and matches for every ngram length with counts from hypo + for (size_t i = 0; i < BleuScoreState::bleu_order; i++) { + m_count_history[i] += ngram_counts[i]; + m_match_history[i] += ngram_matches[i]; + + // do this for last position in batch + if (ref_id == hypos.size() - 1) { + m_count_history[i] *= m_historySmoothing; + m_match_history[i] *= m_historySmoothing; + } + } + + // update counts for reference and target length + m_source_length_history += cur_source_length; + m_target_length_history += hypos[ref_id].size(); + m_ref_length_history += cur_ref_length; + + // do this for last position in batch + if (ref_id == hypos.size() - 1) { + cerr << "Rank " << rank << ", epoch " << epoch << " ,source length history: " << m_source_length_history << " --> " << m_source_length_history * m_historySmoothing << endl; + cerr << "Rank " << rank << ", epoch " << epoch << " ,target length history: " << m_target_length_history << " --> " << m_target_length_history * m_historySmoothing << endl; + m_source_length_history *= m_historySmoothing; + m_target_length_history *= m_historySmoothing; + m_ref_length_history *= m_historySmoothing; + } + } +} + +/* + * Print batch of reference translations + */ +/*void BleuScoreFeature::PrintReferenceLength(const vector<size_t>& ref_ids) { + for (size_t ref_id = 0; ref_id < ref_ids.size(); ++ref_id){ + size_t cur_ref_length = (m_refs[ref_ids[ref_id]].first)[0]; // TODO!! + cerr << "reference length: " << cur_ref_length << endl; + } +}*/ + +size_t BleuScoreFeature::GetClosestRefLength(size_t ref_id, int hypoLength) { + // look for closest reference + int currentDist = -1; + int closestRefLength = -1; + for (size_t i = 0; i < (m_refs[ref_id].first).size(); ++i) { + if (closestRefLength == -1 || abs(hypoLength - (int)(m_refs[ref_id].first)[i]) < currentDist) { + closestRefLength = (m_refs[ref_id].first)[i]; + currentDist = abs(hypoLength - (int)(m_refs[ref_id].first)[i]); + } + } + return (size_t)closestRefLength; +} + +/* + * Given a phrase (current translation) calculate its ngram counts and + * its ngram matches against the ngrams in the reference translation + */ +void BleuScoreFeature::GetNgramMatchCounts(Phrase& phrase, + const NGrams& ref_ngram_counts, + std::vector< size_t >& ret_counts, + std::vector< size_t >& ret_matches, + size_t skip_first) const +{ + NGrams::const_iterator ref_ngram_counts_iter; + size_t ngram_start_idx, ngram_end_idx; + + // Chiang et al (2008) use unclipped counts of ngram matches + for (size_t end_idx = skip_first; end_idx < phrase.GetSize(); end_idx++) { + for (size_t order = 0; order < BleuScoreState::bleu_order; order++) { + if (order > end_idx) break; + + ngram_end_idx = end_idx; + ngram_start_idx = end_idx - order; + + Phrase ngram = phrase.GetSubString(WordsRange(ngram_start_idx, ngram_end_idx), 0); + ret_counts[order]++; + + ref_ngram_counts_iter = ref_ngram_counts.find(ngram); + if (ref_ngram_counts_iter != ref_ngram_counts.end()) + ret_matches[order]++; + } + } +} + +// score ngrams of words that have been added before the previous word span +void BleuScoreFeature::GetNgramMatchCounts_prefix(Phrase& phrase, + const NGrams& ref_ngram_counts, + std::vector< size_t >& ret_counts, + std::vector< size_t >& ret_matches, + size_t new_start_indices, + size_t last_end_index) const +{ + NGrams::const_iterator ref_ngram_counts_iter; + size_t ngram_start_idx, ngram_end_idx; + + // Chiang et al (2008) use unclipped counts of ngram matches + for (size_t start_idx = 0; start_idx < new_start_indices; start_idx++) { + for (size_t order = 0; order < BleuScoreState::bleu_order; order++) { + ngram_start_idx = start_idx; + ngram_end_idx = start_idx + order; + if (order > ngram_end_idx) break; + if (ngram_end_idx > last_end_index) break; + + Phrase ngram = phrase.GetSubString(WordsRange(ngram_start_idx, ngram_end_idx), 0); + ret_counts[order]++; + + ref_ngram_counts_iter = ref_ngram_counts.find(ngram); + if (ref_ngram_counts_iter != ref_ngram_counts.end()) + ret_matches[order]++; + } + } +} + +// score ngrams around the overlap of two previously scored phrases +void BleuScoreFeature::GetNgramMatchCounts_overlap(Phrase& phrase, + const NGrams& ref_ngram_counts, + std::vector< size_t >& ret_counts, + std::vector< size_t >& ret_matches, + size_t overlap_index) const +{ + NGrams::const_iterator ref_ngram_counts_iter; + size_t ngram_start_idx, ngram_end_idx; + + // Chiang et al (2008) use unclipped counts of ngram matches + for (size_t end_idx = overlap_index; end_idx < phrase.GetSize(); end_idx++) { + if (end_idx >= (overlap_index+BleuScoreState::bleu_order-1)) break; + for (size_t order = 0; order < BleuScoreState::bleu_order; order++) { + if (order > end_idx) break; + + ngram_end_idx = end_idx; + ngram_start_idx = end_idx - order; + if (ngram_start_idx >= overlap_index) continue; // only score ngrams that span the overlap point + + Phrase ngram = phrase.GetSubString(WordsRange(ngram_start_idx, ngram_end_idx), 0); + ret_counts[order]++; + + ref_ngram_counts_iter = ref_ngram_counts.find(ngram); + if (ref_ngram_counts_iter != ref_ngram_counts.end()) + ret_matches[order]++; + } + } +} + +void BleuScoreFeature::GetClippedNgramMatchesAndCounts(Phrase& phrase, + const NGrams& ref_ngram_counts, + std::vector< size_t >& ret_counts, + std::vector< size_t >& ret_matches, + size_t skip_first) const +{ + NGrams::const_iterator ref_ngram_counts_iter; + size_t ngram_start_idx, ngram_end_idx; + + Matches ngram_matches; + for (size_t end_idx = skip_first; end_idx < phrase.GetSize(); end_idx++) { + for (size_t order = 0; order < BleuScoreState::bleu_order; order++) { + if (order > end_idx) break; + + ngram_end_idx = end_idx; + ngram_start_idx = end_idx - order; + + Phrase ngram = phrase.GetSubString(WordsRange(ngram_start_idx, ngram_end_idx), 0); + ret_counts[order]++; + + ref_ngram_counts_iter = ref_ngram_counts.find(ngram); + if (ref_ngram_counts_iter != ref_ngram_counts.end()) { + ngram_matches[order][ngram]++; + } + } + } + + // clip ngram matches + for (size_t order = 0; order < BleuScoreState::bleu_order; order++) { + NGrams::const_iterator iter; + + // iterate over ngram counts for every ngram order + for (iter=ngram_matches[order].begin(); iter != ngram_matches[order].end(); ++iter) { + ref_ngram_counts_iter = ref_ngram_counts.find(iter->first); + if (iter->second > ref_ngram_counts_iter->second) { + ret_matches[order] += ref_ngram_counts_iter->second; + } + else { + ret_matches[order] += iter->second; + } + } + } +} + +/* + * Given a previous state, compute Bleu score for the updated state with an additional target + * phrase translated. + */ +FFState* BleuScoreFeature::Evaluate(const Hypothesis& cur_hypo, + const FFState* prev_state, + ScoreComponentCollection* accumulator) const +{ + if (!m_enabled) return new BleuScoreState(); + + NGrams::const_iterator reference_ngrams_iter; + const BleuScoreState& ps = dynamic_cast<const BleuScoreState&>(*prev_state); + BleuScoreState* new_state = new BleuScoreState(ps); + + float old_bleu, new_bleu; + size_t num_new_words, ctx_start_idx, ctx_end_idx; + + // Calculate old bleu; + old_bleu = CalculateBleu(new_state); + + // Get context and append new words. + num_new_words = cur_hypo.GetCurrTargetLength(); + if (num_new_words == 0) { + return new_state; + } + + Phrase new_words = ps.m_words; + new_words.Append(cur_hypo.GetCurrTargetPhrase()); + //cerr << "NW: " << new_words << endl; + + // get ngram matches for new words + GetNgramMatchCounts(new_words, + m_cur_ref_ngrams, + new_state->m_ngram_counts, + new_state->m_ngram_matches, + new_state->m_words.GetSize()); // number of words in previous states + + // Update state variables + ctx_end_idx = new_words.GetSize()-1; + size_t bleu_context_length = BleuScoreState::bleu_order -1; + if (ctx_end_idx > bleu_context_length) { + ctx_start_idx = ctx_end_idx - bleu_context_length; + } else { + ctx_start_idx = 0; + } + + WordsBitmap coverageVector = cur_hypo.GetWordsBitmap(); + new_state->m_source_length = coverageVector.GetNumWordsCovered(); + + new_state->m_words = new_words.GetSubString(WordsRange(ctx_start_idx, + ctx_end_idx)); + new_state->m_target_length += cur_hypo.GetCurrTargetLength(); + + // we need a scaled reference length to compare the current target phrase to the corresponding reference phrase + new_state->m_scaled_ref_length = m_cur_ref_length * + ((float)coverageVector.GetNumWordsCovered()/coverageVector.GetSize()); + + // Calculate new bleu. + new_bleu = CalculateBleu(new_state); + + // Set score to new Bleu score + accumulator->PlusEquals(this, new_bleu - old_bleu); + return new_state; +} + +FFState* BleuScoreFeature::EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, + ScoreComponentCollection* accumulator ) const { + if (!m_enabled) return new BleuScoreState(); + + NGrams::const_iterator reference_ngrams_iter; + + const Phrase& curr_target_phrase = static_cast<const Phrase&>(cur_hypo.GetCurrTargetPhrase()); +// cerr << "\nCur target phrase: " << cur_hypo.GetTargetLHS() << " --> " << curr_target_phrase << endl; + + // Calculate old bleu of previous states + float old_bleu = 0, new_bleu = 0; + size_t num_old_words = 0, num_words_first_prev = 0; + size_t num_words_added_left = 0, num_words_added_right = 0; + + // double-check cases where more than two previous hypotheses were combined + assert(cur_hypo.GetPrevHypos().size() <= 2); + BleuScoreState* new_state; + if (cur_hypo.GetPrevHypos().size() == 0) + new_state = new BleuScoreState(); + else { + const FFState* prev_state_zero = cur_hypo.GetPrevHypo(0)->GetFFState(featureID); + const BleuScoreState& ps_zero = dynamic_cast<const BleuScoreState&>(*prev_state_zero); + new_state = new BleuScoreState(ps_zero); + num_words_first_prev = ps_zero.m_target_length; + + for (size_t i = 0; i < cur_hypo.GetPrevHypos().size(); ++i) { + const FFState* prev_state = cur_hypo.GetPrevHypo(i)->GetFFState(featureID); + const BleuScoreState* ps = dynamic_cast<const BleuScoreState*>(prev_state); + BleuScoreState* ps_nonConst = const_cast<BleuScoreState*>(ps); +// cerr << "prev phrase: " << cur_hypo.GetPrevHypo(i)->GetOutputPhrase() +// << " ( " << cur_hypo.GetPrevHypo(i)->GetTargetLHS() << ")" << endl; + + old_bleu += CalculateBleu(ps_nonConst); + num_old_words += ps->m_target_length; + + if (i > 0) + // add ngram matches from other previous states + new_state->AddNgramCountAndMatches(ps_nonConst->m_ngram_counts, ps_nonConst->m_ngram_matches); + } + } + + // check if we are already done (don't add <s> and </s>) + size_t numWordsCovered = cur_hypo.GetCurrSourceRange().GetNumWordsCovered(); + if (numWordsCovered == m_cur_source_length) { + // Bleu score stays the same, do not need to add anything + //accumulator->PlusEquals(this, 0); + return new_state; + } + + // set new context + Phrase new_words = cur_hypo.GetOutputPhrase(); + new_state->m_words = new_words; + size_t num_curr_words = new_words.GetSize(); + + // get ngram matches for new words + if (num_old_words == 0) { +// cerr << "compute right ngram context" << endl; + GetNgramMatchCounts(new_words, + m_cur_ref_ngrams, + new_state->m_ngram_counts, + new_state->m_ngram_matches, + 0); + } + else if (new_words.GetSize() == num_old_words) { + // two hypotheses were glued together, compute new ngrams on the basis of first hypothesis + num_words_added_right = num_curr_words - num_words_first_prev; + // score around overlap point +// cerr << "compute overlap ngram context (" << (num_words_first_prev) << ")" << endl; + GetNgramMatchCounts_overlap(new_words, + m_cur_ref_ngrams, + new_state->m_ngram_counts, + new_state->m_ngram_matches, + num_words_first_prev); + } + else if (num_old_words + curr_target_phrase.GetNumTerminals() == num_curr_words) { + assert(curr_target_phrase.GetSize() == curr_target_phrase.GetNumTerminals()+1); + // previous hypothesis + rule with 1 non-terminal were combined (NT substituted by Ts) + for (size_t i = 0; i < curr_target_phrase.GetSize(); ++i) + if (curr_target_phrase.GetWord(i).IsNonTerminal()) { + num_words_added_left = i; + num_words_added_right = curr_target_phrase.GetSize() - (i+1); + break; + } + + // left context +// cerr << "compute left ngram context" << endl; + if (num_words_added_left > 0) + GetNgramMatchCounts_prefix(new_words, + m_cur_ref_ngrams, + new_state->m_ngram_counts, + new_state->m_ngram_matches, + num_words_added_left, + num_curr_words - num_words_added_right - 1); + + // right context +// cerr << "compute right ngram context" << endl; + if (num_words_added_right > 0) + GetNgramMatchCounts(new_words, + m_cur_ref_ngrams, + new_state->m_ngram_counts, + new_state->m_ngram_matches, + num_words_added_left + num_old_words); + } + else { + cerr << "undefined state.. " << endl; + exit(1); + } + + // Update state variables + size_t ctx_start_idx = 0; + size_t ctx_end_idx = new_words.GetSize()-1; + size_t bleu_context_length = BleuScoreState::bleu_order -1; + if (ctx_end_idx > bleu_context_length) { + ctx_start_idx = ctx_end_idx - bleu_context_length; + } + + new_state->m_source_length = cur_hypo.GetCurrSourceRange().GetNumWordsCovered(); + new_state->m_words = new_words.GetSubString(WordsRange(ctx_start_idx, ctx_end_idx)); + new_state->m_target_length = cur_hypo.GetOutputPhrase().GetSize(); + + // we need a scaled reference length to compare the current target phrase to the corresponding + // reference phrase + size_t cur_source_length = m_cur_source_length; + new_state->m_scaled_ref_length = m_cur_ref_length * (float(new_state->m_source_length)/cur_source_length); + + // Calculate new bleu. + new_bleu = CalculateBleu(new_state); + + // Set score to new Bleu score + accumulator->PlusEquals(this, new_bleu - old_bleu); + return new_state; +} + +/** + * Calculate real sentence Bleu score of complete translation + */ +float BleuScoreFeature::CalculateBleu(Phrase translation) const +{ + if (translation.GetSize() == 0) + return 0.0; + + Phrase normTranslation = translation; + // remove start and end symbol for chart decoding + if (m_cur_source_length != m_cur_norm_source_length) { + WordsRange* range = new WordsRange(1, translation.GetSize()-2); + normTranslation = translation.GetSubString(*range); + } + + // get ngram matches for translation + BleuScoreState* state = new BleuScoreState(); + GetClippedNgramMatchesAndCounts(normTranslation, + m_cur_ref_ngrams, + state->m_ngram_counts, + state->m_ngram_matches, + 0); // number of words in previous states + + // set state variables + state->m_words = normTranslation; + state->m_source_length = m_cur_norm_source_length; + state->m_target_length = normTranslation.GetSize(); + state->m_scaled_ref_length = m_cur_ref_length; + + // Calculate bleu. + return CalculateBleu(state); +} + +/* + * Calculate Bleu score for a partial hypothesis given as state. + */ +float BleuScoreFeature::CalculateBleu(BleuScoreState* state) const { + if (!state->m_ngram_counts[0]) return 0; + if (!state->m_ngram_matches[0]) return 0; // if we have no unigram matches, score should be 0 + + float precision = 1.0; + float smooth = 1; + float smoothed_count, smoothed_matches; + + if (m_sentence_bleu || m_simple_history_bleu) { + // Calculate geometric mean of modified ngram precisions + // BLEU = BP * exp(SUM_1_4 1/4 * log p_n) + // = BP * 4th root(PRODUCT_1_4 p_n) + for (size_t i = 0; i < BleuScoreState::bleu_order; i++) { + if (state->m_ngram_counts[i]) { + smoothed_matches = state->m_ngram_matches[i]; + smoothed_count = state->m_ngram_counts[i]; + + switch (m_smoothing_scheme) { + case PLUS_ONE: + default: + if (i > 0) { + // smoothing for all n > 1 + smoothed_matches += 1; + smoothed_count += 1; + } + break; + case PLUS_POINT_ONE: + if (i > 0) { + // smoothing for all n > 1 + smoothed_matches += 0.1; + smoothed_count += 0.1; + } + break; + case PAPINENI: + if (state->m_ngram_matches[i] == 0) { + smooth *= 0.5; + smoothed_matches += smooth; + smoothed_count += smooth; + } + break; + } + + if (m_simple_history_bleu) { + smoothed_matches += m_match_history[i]; + smoothed_count += m_count_history[i]; + } + + precision *= smoothed_matches/smoothed_count; + } + } + + // take geometric mean + precision = pow(precision, (float)1/4); + + // Apply brevity penalty if applicable. + // BP = 1 if c > r + // BP = e^(1- r/c)) if c <= r + // where + // c: length of the candidate translation + // r: effective reference length (sum of best match lengths for each candidate sentence) + if (m_simple_history_bleu) { + if ((m_target_length_history + state->m_target_length) < (m_ref_length_history + state->m_scaled_ref_length)) { + float smoothed_target_length = m_target_length_history + state->m_target_length; + float smoothed_ref_length = m_ref_length_history + state->m_scaled_ref_length; + precision *= exp(1 - (smoothed_ref_length/smoothed_target_length)); + } + } + else { + if (state->m_target_length < state->m_scaled_ref_length) { + float target_length = state->m_target_length; + float ref_length = state->m_scaled_ref_length; + precision *= exp(1 - (ref_length/target_length)); + } + } + + //cerr << "precision: " << precision << endl; + + // Approximate bleu score as of Chiang/Resnik is scaled by the size of the input: + // B(e;f,{r_k}) = (O_f + |f|) * BLEU(O + c(e;{r_k})) + // where c(e;) is a vector of reference length, ngram counts and ngram matches + if (m_scale_by_input_length) { + precision *= m_cur_norm_source_length; + } + else if (m_scale_by_avg_input_length) { + precision *= m_avg_input_length; + } + else if (m_scale_by_inverse_length) { + precision *= (100/m_cur_norm_source_length); + } + else if (m_scale_by_avg_inverse_length) { + precision *= (100/m_avg_input_length); + } + + return precision * m_scale_by_x; + } + else { + // Revised history BLEU: compute Bleu in the context of the pseudo-document + // B(b) = size_of_oracle_doc * (Bleu(B_hist + b) - Bleu(B_hist)) + // Calculate geometric mean of modified ngram precisions + // BLEU = BP * exp(SUM_1_4 1/4 * log p_n) + // = BP * 4th root(PRODUCT_1_4 p_n) + for (size_t i = 0; i < BleuScoreState::bleu_order; i++) { + if (state->m_ngram_counts[i]) { + smoothed_matches = m_match_history[i] + state->m_ngram_matches[i] + 0.1; + smoothed_count = m_count_history[i] + state->m_ngram_counts[i] + 0.1; + precision *= smoothed_matches/smoothed_count; + } + } + + // take geometric mean + precision = pow(precision, (float)1/4); + + // Apply brevity penalty if applicable. + if ((m_target_length_history + state->m_target_length) < (m_ref_length_history + state->m_scaled_ref_length)) + precision *= exp(1 - ((m_ref_length_history + state->m_scaled_ref_length)/(m_target_length_history + state->m_target_length))); + + cerr << "precision: " << precision << endl; + + // **BLEU score of pseudo-document** + float precision_pd = 1.0; + if (m_target_length_history > 0) { + for (size_t i = 0; i < BleuScoreState::bleu_order; i++) + if (m_count_history[i] != 0) + precision_pd *= (m_match_history[i] + 0.1)/(m_count_history[i] + 0.1); + + // take geometric mean + precision_pd = pow(precision_pd, (float)1/4); + + // Apply brevity penalty if applicable. + if (m_target_length_history < m_ref_length_history) + precision_pd *= exp(1 - (m_ref_length_history/m_target_length_history)); + } + else + precision_pd = 0; + // **end BLEU of pseudo-document** + + cerr << "precision pd: " << precision_pd << endl; + + float sentence_impact; + if (m_target_length_history > 0) + sentence_impact = m_target_length_history * (precision - precision_pd); + else + sentence_impact = precision; + + cerr << "sentence impact: " << sentence_impact << endl; + return sentence_impact * m_scale_by_x; + } +} + +const FFState* BleuScoreFeature::EmptyHypothesisState(const InputType& input) const +{ + return new BleuScoreState(); +} + +} // namespace. + diff --git a/moses/src/BleuScoreFeature.h b/moses/src/BleuScoreFeature.h new file mode 100644 index 000000000..4803fd75a --- /dev/null +++ b/moses/src/BleuScoreFeature.h @@ -0,0 +1,176 @@ +#ifndef BLUESCOREFEATURE_H +#define BLUESCOREFEATURE_H + +#include <utility> +#include <string> +#include <vector> + +#include <boost/unordered_map.hpp> + +#include "FeatureFunction.h" + +#include "FFState.h" +#include "Phrase.h" +#include "ChartHypothesis.h" + +namespace Moses { + +class BleuScoreFeature; + +class BleuScoreState : public FFState { +public: + friend class BleuScoreFeature; + static size_t bleu_order; + + BleuScoreState(); + virtual int Compare(const FFState& other) const; + void print(std::ostream& out) const; + +private: + Phrase m_words; + size_t m_source_length; + size_t m_target_length; + + // scaled reference length is needed for scoring incomplete hypotheses against reference translation + float m_scaled_ref_length; + + std::vector< size_t > m_ngram_counts; + std::vector< size_t > m_ngram_matches; + + void AddNgramCountAndMatches(std::vector< size_t >& counts, std::vector< size_t >& matches); +}; + + +std::ostream& operator<<(std::ostream& out, const BleuScoreState& state); + + +class BleuScoreFeature : public StatefulFeatureFunction { +public: + + typedef boost::unordered_map< Phrase, size_t > NGrams; + typedef boost::unordered_map<size_t, std::pair<std::vector<size_t>,NGrams> > RefCounts; + typedef boost::unordered_map<size_t, NGrams> Matches; + + BleuScoreFeature(): + StatefulFeatureFunction("BleuScore",1), + m_enabled(true), + m_sentence_bleu(true), + m_simple_history_bleu(false), + m_count_history(BleuScoreState::bleu_order), + m_match_history(BleuScoreState::bleu_order), + m_source_length_history(0), + m_target_length_history(0), + m_ref_length_history(0), + m_scale_by_input_length(true), + m_scale_by_avg_input_length(false), + m_scale_by_inverse_length(false), + m_scale_by_avg_inverse_length(false), + m_scale_by_x(1), + m_historySmoothing(0.9), + m_smoothing_scheme(PLUS_POINT_ONE) {} + + std::string GetScoreProducerDescription() const + { + return "BleuScoreFeature"; + } + + std::string GetScoreProducerWeightShortName(unsigned) const + { + return "bl"; + } + + void PrintHistory(std::ostream& out) const; + void LoadReferences(const std::vector< std::vector< std::string > > &); + void SetCurrSourceLength(size_t); + void SetCurrNormSourceLength(size_t); + void SetCurrShortestRefLength(size_t); + void SetCurrAvgRefLength(size_t sent_id); + void SetAvgInputLength (float l) { m_avg_input_length = l; } + void SetCurrReferenceNgrams(size_t sent_id); + size_t GetShortestRefIndex(size_t ref_id); + size_t GetClosestRefLength(size_t ref_id, int hypoLength); + void UpdateHistory(const std::vector< const Word* >&); + void UpdateHistory(const std::vector< std::vector< const Word* > >& hypos, std::vector<size_t>& sourceLengths, std::vector<size_t>& ref_ids, size_t rank, size_t epoch); + void PrintRefLength(const std::vector<size_t>& ref_ids); + void SetBleuParameters(bool disable, bool sentenceBleu, bool scaleByInputLength, bool scaleByAvgInputLength, + bool scaleByInverseLength, bool scaleByAvgInverseLength, + float scaleByX, float historySmoothing, size_t scheme, bool simpleHistoryBleu); + + void GetNgramMatchCounts(Phrase&, + const NGrams&, + std::vector< size_t >&, + std::vector< size_t >&, + size_t skip = 0) const; + void GetNgramMatchCounts_prefix(Phrase&, + const NGrams&, + std::vector< size_t >&, + std::vector< size_t >&, + size_t new_start_indices, + size_t last_end_index) const; + void GetNgramMatchCounts_overlap(Phrase& phrase, + const NGrams& ref_ngram_counts, + std::vector< size_t >& ret_counts, + std::vector< size_t >& ret_matches, + size_t overlap_index) const; + void GetClippedNgramMatchesAndCounts(Phrase&, + const NGrams&, + std::vector< size_t >&, + std::vector< size_t >&, + size_t skip = 0) const; + + FFState* Evaluate( const Hypothesis& cur_hypo, + const FFState* prev_state, + ScoreComponentCollection* accumulator) const; + FFState* EvaluateChart(const ChartHypothesis& cur_hypo, + int featureID, + ScoreComponentCollection* accumulator) const; + bool Enabled() const { return m_enabled; } + float CalculateBleu(BleuScoreState*) const; + float CalculateBleu(Phrase translation) const; + const FFState* EmptyHypothesisState(const InputType&) const; + + float GetSourceLengthHistory() { return m_source_length_history; } + float GetTargetLengthHistory() { return m_target_length_history; } + float GetAverageInputLength() { return m_avg_input_length; } + +private: + bool m_enabled; + bool m_sentence_bleu; + bool m_simple_history_bleu; + + // counts for pseudo-document + std::vector< float > m_count_history; + std::vector< float > m_match_history; + float m_source_length_history; + float m_target_length_history; + float m_ref_length_history; + + size_t m_cur_source_length; + size_t m_cur_norm_source_length; // length without <s>, </s> + RefCounts m_refs; + NGrams m_cur_ref_ngrams; + float m_cur_ref_length; + + // scale BLEU score by history of input length + bool m_scale_by_input_length; + bool m_scale_by_avg_input_length; + + // scale by the inverse of the input length * 100 + bool m_scale_by_inverse_length; + bool m_scale_by_avg_inverse_length; + + float m_avg_input_length; + + float m_scale_by_x; + + // smoothing factor for history counts + float m_historySmoothing; + + enum SmoothingScheme { PLUS_ONE = 1, PLUS_POINT_ONE = 2, PAPINENI = 3 }; + SmoothingScheme m_smoothing_scheme; +}; + +} // Namespace. + +#endif //BLUESCOREFEATURE_H + diff --git a/moses/src/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp b/moses/src/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp index f8270d113..80e508176 100644 --- a/moses/src/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp +++ b/moses/src/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp @@ -41,7 +41,6 @@ ChartRuleLookupManagerOnDisk::ChartRuleLookupManagerOnDisk( const WordPenaltyProducer *wpProducer, const std::vector<FactorType> &inputFactorsVec, const std::vector<FactorType> &outputFactorsVec, - const std::vector<float> &weight, const std::string &filePath) : ChartRuleLookupManagerCYKPlus(sentence, cellColl) , m_dictionary(dictionary) @@ -50,7 +49,6 @@ ChartRuleLookupManagerOnDisk::ChartRuleLookupManagerOnDisk( , m_wpProducer(wpProducer) , m_inputFactorsVec(inputFactorsVec) , m_outputFactorsVec(outputFactorsVec) - , m_weight(weight) , m_filePath(filePath) { CHECK(m_expandableDottedRuleListVec.size() == 0); @@ -83,6 +81,7 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection( const WordsRange &range, ChartTranslationOptionList &outColl) { + const StaticData &staticData = StaticData::Instance(); size_t relEndPos = range.GetEndPos() - range.GetStartPos(); size_t absEndPos = range.GetEndPos(); @@ -237,11 +236,12 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection( const OnDiskPt::TargetPhraseCollection *tpcollBerkeleyDb = node->GetTargetPhraseCollection(m_dictionary.GetTableLimit(), m_dbWrapper); + std::vector<float> weightT = staticData.GetWeights(m_dictionary.GetFeature()); targetPhraseCollection = tpcollBerkeleyDb->ConvertToMoses(m_inputFactorsVec ,m_outputFactorsVec ,m_dictionary - ,m_weight + ,weightT ,m_wpProducer ,*m_languageModels ,m_filePath diff --git a/moses/src/CYKPlusParser/ChartRuleLookupManagerOnDisk.h b/moses/src/CYKPlusParser/ChartRuleLookupManagerOnDisk.h index 4d629405d..333d975cc 100644 --- a/moses/src/CYKPlusParser/ChartRuleLookupManagerOnDisk.h +++ b/moses/src/CYKPlusParser/ChartRuleLookupManagerOnDisk.h @@ -44,7 +44,6 @@ class ChartRuleLookupManagerOnDisk : public ChartRuleLookupManagerCYKPlus const WordPenaltyProducer *wpProducer, const std::vector<FactorType> &inputFactorsVec, const std::vector<FactorType> &outputFactorsVec, - const std::vector<float> &weight, const std::string &filePath); ~ChartRuleLookupManagerOnDisk(); @@ -59,7 +58,6 @@ class ChartRuleLookupManagerOnDisk : public ChartRuleLookupManagerCYKPlus const WordPenaltyProducer *m_wpProducer; const std::vector<FactorType> &m_inputFactorsVec; const std::vector<FactorType> &m_outputFactorsVec; - const std::vector<float> &m_weight; const std::string &m_filePath; std::vector<DottedRuleStackOnDisk*> m_expandableDottedRuleListVec; std::map<UINT64, const TargetPhraseCollection*> m_cache; diff --git a/moses/src/ChartHypothesis.cpp b/moses/src/ChartHypothesis.cpp index 7d30bf7a6..24d9838db 100644 --- a/moses/src/ChartHypothesis.cpp +++ b/moses/src/ChartHypothesis.cpp @@ -32,6 +32,8 @@ #include "ChartTranslationOptions.h" #include "FFState.h" +using namespace std; + namespace Moses { @@ -90,14 +92,12 @@ ChartHypothesis::~ChartHypothesis() */ void ChartHypothesis::CreateOutputPhrase(Phrase &outPhrase) const { - const AlignmentInfo::NonTermIndexMap &nonTermIndexMap = - GetCurrTargetPhrase().GetAlignmentInfo().GetNonTermIndexMap(); for (size_t pos = 0; pos < GetCurrTargetPhrase().GetSize(); ++pos) { const Word &word = GetCurrTargetPhrase().GetWord(pos); if (word.IsNonTerminal()) { // non-term. fill out with prev hypo - size_t nonTermInd = nonTermIndexMap[pos]; + size_t nonTermInd = GetCurrTargetPhrase().GetAlignmentInfo().GetNonTermIndexMap()[pos]; const ChartHypothesis *prevHypo = m_prevHypos[nonTermInd]; prevHypo->CreateOutputPhrase(outPhrase); } @@ -160,21 +160,21 @@ void ChartHypothesis::CalcScore() const ScoreComponentCollection &scoreBreakdown = GetCurrTargetPhrase().GetScoreBreakdown(); m_scoreBreakdown.PlusEquals(scoreBreakdown); + //Add pre-computed features + m_manager.InsertPreCalculatedScores(GetCurrTargetPhrase(), &m_scoreBreakdown); + // compute values of stateless feature functions that were not // cached in the translation option-- there is no principled distinction - - //const vector<const StatelessFeatureFunction*>& sfs = - // m_manager.GetTranslationSystem()->GetStatelessFeatureFunctions(); - // TODO! - //for (unsigned i = 0; i < sfs.size(); ++i) { - // sfs[i]->ChartEvaluate(m_targetPhrase, &m_scoreBreakdown); - //} + const std::vector<const StatelessFeatureFunction*>& sfs = + m_manager.GetTranslationSystem()->GetStatelessFeatureFunctions(); + for (unsigned i = 0; i < sfs.size(); ++i) + if (sfs[i]->ComputeValueInTranslationOption() == false) + sfs[i]->EvaluateChart(ChartBasedFeatureContext(this),&m_scoreBreakdown); const std::vector<const StatefulFeatureFunction*>& ffs = m_manager.GetTranslationSystem()->GetStatefulFeatureFunctions(); - for (unsigned i = 0; i < ffs.size(); ++i) { - m_ffStates[i] = ffs[i]->EvaluateChart(*this,i,&m_scoreBreakdown); - } + for (unsigned i = 0; i < ffs.size(); ++i) + m_ffStates[i] = ffs[i]->EvaluateChart(*this,i,&m_scoreBreakdown); m_totalScore = m_scoreBreakdown.GetWeightedScore(); } diff --git a/moses/src/ChartManager.cpp b/moses/src/ChartManager.cpp index d9a157f34..adb11ea94 100644 --- a/moses/src/ChartManager.cpp +++ b/moses/src/ChartManager.cpp @@ -23,6 +23,7 @@ #include "ChartManager.h" #include "ChartCell.h" #include "ChartHypothesis.h" +#include "ChartTranslationOptions.h" #include "ChartTrellisDetourQueue.h" #include "ChartTrellisNode.h" #include "ChartTrellisPath.h" @@ -197,21 +198,16 @@ void ChartManager::CalcNBest(size_t count, ChartTrellisPathList &ret,bool onlyDi boost::shared_ptr<ChartTrellisPath> basePath(new ChartTrellisPath(*hypo)); // Add it to the n-best list. - ret.Add(basePath); if (count == 1) { + ret.Add(basePath); return; } - // Record the output phrase if distinct translations are required. - set<Phrase> distinctHyps; - if (onlyDistinct) { - distinctHyps.insert(basePath->GetOutputPhrase()); - } - // Set a limit on the number of detours to pop. If the n-best list is // restricted to distinct translations then this limit should be bigger // than n. The n-best factor determines how much bigger the limit should be. - const size_t nBestFactor = StaticData::Instance().GetNBestFactor(); + const StaticData &staticData = StaticData::Instance(); + const size_t nBestFactor = staticData.GetNBestFactor(); size_t popLimit; if (!onlyDistinct) { popLimit = count-1; @@ -227,36 +223,48 @@ void ChartManager::CalcNBest(size_t count, ChartTrellisPathList &ret,bool onlyDi // contain no more than popLimit items. ChartTrellisDetourQueue contenders(popLimit); - // Create a ChartTrellisDetour for each single-point deviation from basePath - // and add them to the queue. - CreateDeviantPaths(basePath, contenders); - + // Get all complete translations + Word *w = new Word(); + //w->CreateFromString(Output, staticData.GetOutputFactorOrder(), "TOP", true); + w->CreateFromString(Output, staticData.GetOutputFactorOrder(), "S", true); + const HypoList topHypos = *(lastCell.GetSortedHypotheses(*w)); + + // Create a ChartTrellisDetour for each complete translation and add it to the queue + for (size_t i=0; i<topHypos.size(); ++i) { + const ChartHypothesis &hypo = *(topHypos[i]); + boost::shared_ptr<ChartTrellisPath> basePath(new ChartTrellisPath(hypo)); + ChartTrellisDetour *detour = new ChartTrellisDetour(basePath, basePath->GetFinalNode(), hypo); + contenders.Push(detour); + } + + // Record the output phrase if distinct translations are required. + set<Phrase> distinctHyps; + // MAIN loop - for (size_t i = 0; - ret.GetSize() < count && !contenders.Empty() && i < popLimit; - ++i) { + for (size_t i = 0; ret.GetSize() < count && !contenders.Empty() && i < popLimit; ++i) { // Get the best detour from the queue. std::auto_ptr<const ChartTrellisDetour> detour(contenders.Pop()); CHECK(detour.get()); // Create a full base path from the chosen detour. - basePath.reset(new ChartTrellisPath(*detour)); - + //basePath.reset(new ChartTrellisPath(*detour)); + boost::shared_ptr<ChartTrellisPath> path(new ChartTrellisPath(*detour)); + // Generate new detours from this base path and add them to the queue of // contenders. The new detours deviate from the base path by a single // replacement along the previous detour sub-path. - CHECK(basePath->GetDeviationPoint()); - CreateDeviantPaths(basePath, *(basePath->GetDeviationPoint()), contenders); + CHECK(path->GetDeviationPoint()); + CreateDeviantPaths(path, *(path->GetDeviationPoint()), contenders); // If the n-best list is allowed to contain duplicate translations (at the // surface level) then add the new path unconditionally, otherwise check // whether the translation has seen before. if (!onlyDistinct) { - ret.Add(basePath); + ret.Add(path); } else { - Phrase tgtPhrase = basePath->GetOutputPhrase(); + Phrase tgtPhrase = path->GetOutputPhrase(); if (distinctHyps.insert(tgtPhrase).second) { - ret.Add(basePath); + ret.Add(path); } } } @@ -375,6 +383,7 @@ void ChartManager::CreateTranslationOptionsForRange(const WordsRange &wordsRange } m_translationOptionList.ApplyThreshold(); + PreCalculateScores(); } //! special handling of ONE unknown words. @@ -435,7 +444,7 @@ void ChartManager::ProcessOneUnknownWord(const Word &sourceWord, const WordsRang //targetPhrase->SetScore(); targetPhrase->SetScore(unknownWordPenaltyProducer, unknownScore); targetPhrase->SetScore(m_system->GetWordPenaltyProducer(), wordPenaltyScore); - targetPhrase->SetSourcePhrase(m_unksrc); + targetPhrase->SetSourcePhrase(*m_unksrc); targetPhrase->SetTargetLHS(targetLHS); // chart rule @@ -460,7 +469,7 @@ void ChartManager::ProcessOneUnknownWord(const Word &sourceWord, const WordsRang CHECK(targetLHS.GetFactor(0) != NULL); m_cacheTargetPhraseCollection.push_back(tpc); - targetPhrase->SetSourcePhrase(m_unksrc); + targetPhrase->SetSourcePhrase(*m_unksrc); targetPhrase->SetScore(unknownWordPenaltyProducer, unknownScore); targetPhrase->SetTargetLHS(targetLHS); @@ -469,5 +478,42 @@ void ChartManager::ProcessOneUnknownWord(const Word &sourceWord, const WordsRang } } } + +void ChartManager::PreCalculateScores() +{ + for (size_t i = 0; i < m_translationOptionList.GetSize(); ++i) { + const ChartTranslationOptions& cto = m_translationOptionList.Get(i); + for (TargetPhraseCollection::const_iterator j = cto.GetTargetPhraseCollection().begin(); + j != cto.GetTargetPhraseCollection().end(); ++j) { + const TargetPhrase* targetPhrase = *j; + if (m_precalculatedScores.find(*targetPhrase) == m_precalculatedScores.end()) { + ChartBasedFeatureContext context(*targetPhrase,m_source); + const vector<const StatelessFeatureFunction*>& sfs = + m_system->GetStatelessFeatureFunctions(); + ScoreComponentCollection& breakdown = m_precalculatedScores[*targetPhrase]; + for (size_t k = 0; k < sfs.size(); ++k) { + if (!sfs[k]->ComputeValueInTranslationTable()) { + sfs[k]->EvaluateChart(context,&breakdown); + } + } + } + } + } +} + +void ChartManager::InsertPreCalculatedScores( + const TargetPhrase& targetPhrase, ScoreComponentCollection* scoreBreakdown) const +{ + boost::unordered_map<TargetPhrase,ScoreComponentCollection>::const_iterator scoreIter = + m_precalculatedScores.find(targetPhrase); + if (scoreIter != m_precalculatedScores.end()) { + scoreBreakdown->PlusEquals(scoreIter->second); + } else { + TRACE_ERR("ERROR: " << targetPhrase << " missing from precalculation cache" << endl); + assert(0); + } + +} + } // namespace Moses diff --git a/moses/src/ChartManager.h b/moses/src/ChartManager.h index acf5de275..45db8c206 100644 --- a/moses/src/ChartManager.h +++ b/moses/src/ChartManager.h @@ -22,6 +22,7 @@ #pragma once #include <vector> +#include <boost/unordered_map.hpp> #include "ChartCell.h" #include "ChartCellCollection.h" #include "InputType.h" @@ -70,6 +71,12 @@ private: std::vector <DecodeGraph*> m_decodeGraphList; StackVec m_emptyStackVec; + //! Some features should be calculated prior to search + boost::unordered_map<TargetPhrase,ScoreComponentCollection, RuleHash, RuleComparator> m_precalculatedScores; + + //! Pre-calculate most stateless feature values + void PreCalculateScores(); + public: ChartManager(InputType const& source, const TranslationSystem* system); ~ChartManager(); @@ -109,6 +116,11 @@ public: //! contigious hypo id for each input sentence. For debugging purposes unsigned GetNextHypoId() { return m_hypothesisId++; } + + //! Access the pre-calculated values + void InsertPreCalculatedScores(const TargetPhrase& targetPhrase, + ScoreComponentCollection* scoreBreakdown) const; + }; } diff --git a/moses/src/CompactPT/PhraseDecoder.cpp b/moses/src/CompactPT/PhraseDecoder.cpp index 8e998504c..9a4b83b62 100644 --- a/moses/src/CompactPT/PhraseDecoder.cpp +++ b/moses/src/CompactPT/PhraseDecoder.cpp @@ -292,7 +292,7 @@ TargetPhraseVectorPtr PhraseDecoder::DecodeCollection( tpv->push_back(TargetPhrase(Output)); targetPhrase = &tpv->back(); - targetPhrase->SetSourcePhrase(&sourcePhrase); + targetPhrase->SetSourcePhrase(sourcePhrase); alignment.clear(); scores.clear(); @@ -431,7 +431,7 @@ TargetPhraseVectorPtr PhraseDecoder::DecodeCollection( if(scores.size() == m_numScoreComponent) { - targetPhrase->SetScore(m_feature, scores, *m_weight, m_weightWP, *m_languageModels); + targetPhrase->SetScore(m_feature, scores, ScoreComponentCollection() /*sparse*/,*m_weight, m_weightWP, *m_languageModels); if(m_containsAlignmentInfo) state = Alignment; diff --git a/moses/src/DecodeFeature.cpp b/moses/src/DecodeFeature.cpp index f17f8c7c2..67999384a 100644 --- a/moses/src/DecodeFeature.cpp +++ b/moses/src/DecodeFeature.cpp @@ -30,10 +30,10 @@ using namespace std; namespace Moses { -DecodeFeature::DecodeFeature(const std::vector<FactorType> &input, const std::vector<FactorType> &output) : +DecodeFeature::DecodeFeature(const std::string& description, size_t numScoreComponents, const std::vector<FactorType> &input, const std::vector<FactorType> &output) : + StatelessFeatureFunction(description,numScoreComponents), m_input(input), m_output(output) { - m_inputFactors = FactorMask(input); m_outputFactors = FactorMask(output); VERBOSE(2,"DecodeFeature: input=" << m_inputFactors << " output=" << m_outputFactors << std::endl); diff --git a/moses/src/DecodeFeature.h b/moses/src/DecodeFeature.h index e65f2d8b9..2d338979d 100644 --- a/moses/src/DecodeFeature.h +++ b/moses/src/DecodeFeature.h @@ -35,26 +35,28 @@ namespace Moses * A feature on the decoding path (Generation or Translation) * @todo don't quite understand what this is **/ -class DecodeFeature : public StatelessFeatureFunction -{ - -public: - DecodeFeature(const std::vector<FactorType> &input, const std::vector<FactorType> &output); - - //! returns output factor types as specified by the ini file - const FactorMask& GetOutputFactorMask() const; - - //! returns input factor types as specified by the ini file - const FactorMask& GetInputFactorMask() const; - - const std::vector<FactorType>& GetInput() const; - const std::vector<FactorType>& GetOutput() const; +class DecodeFeature : public StatelessFeatureFunction { -private: - std::vector<FactorType> m_input; - std::vector<FactorType> m_output; - FactorMask m_inputFactors; - FactorMask m_outputFactors; + public: + DecodeFeature( const std::string& description, + size_t numScoreComponents, + const std::vector<FactorType> &input, + const std::vector<FactorType> &output); + + //! returns output factor types as specified by the ini file + const FactorMask& GetOutputFactorMask() const; + + //! returns input factor types as specified by the ini file + const FactorMask& GetInputFactorMask() const; + + const std::vector<FactorType>& GetInput() const; + const std::vector<FactorType>& GetOutput() const; + + private: + std::vector<FactorType> m_input; + std::vector<FactorType> m_output; + FactorMask m_inputFactors; + FactorMask m_outputFactors; }; } diff --git a/moses/src/DummyScoreProducers.cpp b/moses/src/DummyScoreProducers.cpp index 545fa86b9..5808363c8 100644 --- a/moses/src/DummyScoreProducers.cpp +++ b/moses/src/DummyScoreProducers.cpp @@ -38,20 +38,6 @@ const FFState* DistortionScoreProducer::EmptyHypothesisState(const InputType &in NOT_FOUND); } -DistortionScoreProducer::DistortionScoreProducer(ScoreIndexManager &scoreIndexManager) -{ - scoreIndexManager.AddScoreProducer(this); -} - -size_t DistortionScoreProducer::GetNumScoreComponents() const -{ - return 1; -} - -std::string DistortionScoreProducer::GetScoreProducerDescription(unsigned) const -{ - return "Distortion"; -} std::string DistortionScoreProducer::GetScoreProducerWeightShortName(unsigned) const { @@ -102,10 +88,6 @@ float DistortionScoreProducer::CalculateDistortionScore(const Hypothesis& hypo, } } -size_t DistortionScoreProducer::GetNumInputScores() const -{ - return 0; -} FFState* DistortionScoreProducer::Evaluate( const Hypothesis& hypo, @@ -126,64 +108,33 @@ FFState* DistortionScoreProducer::Evaluate( } -WordPenaltyProducer::WordPenaltyProducer(ScoreIndexManager &scoreIndexManager) -{ - scoreIndexManager.AddScoreProducer(this); -} - -size_t WordPenaltyProducer::GetNumScoreComponents() const -{ - return 1; -} - -std::string WordPenaltyProducer::GetScoreProducerDescription(unsigned) const -{ - return "WordPenalty"; -} - std::string WordPenaltyProducer::GetScoreProducerWeightShortName(unsigned) const { return "w"; } -size_t WordPenaltyProducer::GetNumInputScores() const -{ - return 0; -} - -void WordPenaltyProducer::Evaluate(const TargetPhrase& tp, ScoreComponentCollection* out) const +void WordPenaltyProducer::Evaluate( + const PhraseBasedFeatureContext& context, + ScoreComponentCollection* out) const { + const TargetPhrase& tp = context.GetTargetPhrase(); out->PlusEquals(this, -static_cast<float>(tp.GetSize())); } -UnknownWordPenaltyProducer::UnknownWordPenaltyProducer(ScoreIndexManager &scoreIndexManager) -{ - scoreIndexManager.AddScoreProducer(this); -} - -size_t UnknownWordPenaltyProducer::GetNumScoreComponents() const -{ - return 1; -} - -std::string UnknownWordPenaltyProducer::GetScoreProducerDescription(unsigned) const -{ - return "!UnknownWordPenalty"; -} - std::string UnknownWordPenaltyProducer::GetScoreProducerWeightShortName(unsigned) const { return "u"; } -size_t UnknownWordPenaltyProducer::GetNumInputScores() const -{ - return 0; -} bool UnknownWordPenaltyProducer::ComputeValueInTranslationOption() const { return true; } +std::string MetaFeatureProducer::GetScoreProducerWeightShortName(unsigned) const +{ + return "m"+m_shortName; +} + } diff --git a/moses/src/DummyScoreProducers.h b/moses/src/DummyScoreProducers.h index fe0c7fdcf..612d63926 100644 --- a/moses/src/DummyScoreProducers.h +++ b/moses/src/DummyScoreProducers.h @@ -15,16 +15,12 @@ class WordsRange; class DistortionScoreProducer : public StatefulFeatureFunction { public: - DistortionScoreProducer(ScoreIndexManager &scoreIndexManager); + DistortionScoreProducer() : StatefulFeatureFunction("Distortion", 1) {} float CalculateDistortionScore(const Hypothesis& hypo, const WordsRange &prev, const WordsRange &curr, const int FirstGapPosition) const; - size_t GetNumScoreComponents() const; - std::string GetScoreProducerDescription(unsigned) const; - std::string GetScoreProducerWeightShortName(unsigned) const; - size_t GetNumInputScores() const; - + std::string GetScoreProducerWeightShortName(unsigned) const; virtual const FFState* EmptyHypothesisState(const InputType &input) const; virtual FFState* Evaluate( @@ -33,8 +29,8 @@ public: ScoreComponentCollection* accumulator) const; virtual FFState* EvaluateChart( - const ChartHypothesis&, - int /* featureID */, + const ChartHypothesis& /* cur_hypo */, + int /* featureID - used to index the state in the previous hypotheses */, ScoreComponentCollection*) const { CHECK(0); // feature function not valid in chart decoder return NULL; @@ -47,31 +43,67 @@ public: class WordPenaltyProducer : public StatelessFeatureFunction { public: - WordPenaltyProducer(ScoreIndexManager &scoreIndexManager); + WordPenaltyProducer() : StatelessFeatureFunction("WordPenalty",1) {} - size_t GetNumScoreComponents() const; - std::string GetScoreProducerDescription(unsigned) const; - std::string GetScoreProducerWeightShortName(unsigned) const; - size_t GetNumInputScores() const; + std::string GetScoreProducerWeightShortName(unsigned) const; virtual void Evaluate( - const TargetPhrase& phrase, - ScoreComponentCollection* out) const; + const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const; + + virtual void EvaluateChart( + const ChartBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const + { + //required but does nothing. + } + + + }; /** unknown word penalty */ class UnknownWordPenaltyProducer : public StatelessFeatureFunction { public: - UnknownWordPenaltyProducer(ScoreIndexManager &scoreIndexManager); + UnknownWordPenaltyProducer() : StatelessFeatureFunction("!UnknownWordPenalty",1) {} - size_t GetNumScoreComponents() const; - std::string GetScoreProducerDescription(unsigned) const; - std::string GetScoreProducerWeightShortName(unsigned) const; - size_t GetNumInputScores() const; + std::string GetScoreProducerWeightShortName(unsigned) const; virtual bool ComputeValueInTranslationOption() const; + void Evaluate( const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const + { + //do nothing - not a real feature + } + + void EvaluateChart( + const ChartBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const + { + //do nothing - not a real feature + } + +}; + +class MetaFeatureProducer : public StatelessFeatureFunction +{ + public: + MetaFeatureProducer(std::string shortName) : StatelessFeatureFunction("MetaFeature_"+shortName,1), m_shortName(shortName) {} + + std::string m_shortName; + + std::string GetScoreProducerWeightShortName(unsigned) const; + + void Evaluate(const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const { + //do nothing - not a real feature + } + void EvaluateChart(const ChartBasedFeatureContext& context, + ScoreComponentCollection*) const { + //do nothing - not a real feature + } }; } diff --git a/moses/src/FFState.h b/moses/src/FFState.h index 527738723..4b1b145a3 100644 --- a/moses/src/FFState.h +++ b/moses/src/FFState.h @@ -17,5 +17,13 @@ public: virtual int Compare(const FFState& other) const = 0; }; +class DummyState : public FFState { +public: + DummyState() {} + int Compare(const FFState& other) const { + return 0; + } +}; + } #endif diff --git a/moses/src/FeatureFunction.cpp b/moses/src/FeatureFunction.cpp index ad9db5e1c..0d70bd70f 100644 --- a/moses/src/FeatureFunction.cpp +++ b/moses/src/FeatureFunction.cpp @@ -1,26 +1,89 @@ -#include "FeatureFunction.h" +#include <stdexcept> #include "util/check.hh" +#include "ChartHypothesis.h" +#include "ChartManager.h" +#include "FeatureFunction.h" +#include "Hypothesis.h" +#include "Manager.h" +#include "TranslationOption.h" + + namespace Moses { +PhraseBasedFeatureContext::PhraseBasedFeatureContext(const Hypothesis* hypothesis) : + m_hypothesis(hypothesis), + m_translationOption(m_hypothesis->GetTranslationOption()), + m_source(m_hypothesis->GetManager().GetSource()) {} + +PhraseBasedFeatureContext::PhraseBasedFeatureContext + (const TranslationOption& translationOption, const InputType& source) : + m_hypothesis(NULL), + m_translationOption(translationOption), + m_source(source) {} + +const TranslationOption& PhraseBasedFeatureContext::GetTranslationOption() const +{ + return m_translationOption; +} + +const InputType& PhraseBasedFeatureContext::GetSource() const +{ + return m_source; +} + +const TargetPhrase& PhraseBasedFeatureContext::GetTargetPhrase() const +{ + return m_translationOption.GetTargetPhrase(); +} + +const WordsBitmap& PhraseBasedFeatureContext::GetWordsBitmap() const +{ + if (!m_hypothesis) { + throw std::logic_error("Coverage vector not available during pre-calculation"); + } + return m_hypothesis->GetWordsBitmap(); +} + + +ChartBasedFeatureContext::ChartBasedFeatureContext + (const ChartHypothesis* hypothesis): + m_hypothesis(hypothesis), + m_targetPhrase(hypothesis->GetCurrTargetPhrase()), + m_source(hypothesis->GetManager().GetSource()) {} + +ChartBasedFeatureContext::ChartBasedFeatureContext( + const TargetPhrase& targetPhrase, + const InputType& source): + m_hypothesis(NULL), + m_targetPhrase(targetPhrase), + m_source(source) {} + +const InputType& ChartBasedFeatureContext::GetSource() const +{ + return m_source; +} + +const TargetPhrase& ChartBasedFeatureContext::GetTargetPhrase() const +{ + return m_targetPhrase; +} + + + FeatureFunction::~FeatureFunction() {} bool StatelessFeatureFunction::IsStateless() const { return true; } + bool StatelessFeatureFunction::ComputeValueInTranslationOption() const { return false; } -void StatelessFeatureFunction::Evaluate( - const TargetPhrase& /* cur_hypo */, - ScoreComponentCollection* /* accumulator */) const -{ - CHECK(!"Please implement Evaluate or set ComputeValueInTranslationOption to true"); -} bool StatefulFeatureFunction::IsStateless() const { diff --git a/moses/src/FeatureFunction.h b/moses/src/FeatureFunction.h index 139725a1a..b1b97bd7e 100644 --- a/moses/src/FeatureFunction.h +++ b/moses/src/FeatureFunction.h @@ -9,11 +9,66 @@ namespace Moses { class TargetPhrase; +class TranslationOption; class Hypothesis; class ChartHypothesis; class FFState; class InputType; class ScoreComponentCollection; +class WordsBitmap; +class WordsRange; + + +/** + * Contains all that a feature function can access without affecting recombination. + * For stateless features, this is all that it can access. Currently this is not + * used for stateful features, as it would need to be retro-fitted to the LM feature. + * TODO: Expose source segmentation,lattice path. + * XXX Don't add anything to the context that would break recombination XXX + **/ +class PhraseBasedFeatureContext +{ + // The context either has a hypothesis (during search), or a TranslationOption and + // source sentence (during pre-calculation). + const Hypothesis* m_hypothesis; + const TranslationOption& m_translationOption; + const InputType& m_source; + +public: + PhraseBasedFeatureContext(const Hypothesis* hypothesis); + PhraseBasedFeatureContext(const TranslationOption& translationOption, + const InputType& source); + + const TranslationOption& GetTranslationOption() const; + const InputType& GetSource() const; + const TargetPhrase& GetTargetPhrase() const; //convenience method + const WordsBitmap& GetWordsBitmap() const; + +}; + +/** + * Same as PhraseBasedFeatureContext, but for chart-based Moses. + **/ +class ChartBasedFeatureContext +{ + //The context either has a hypothesis (during search) or a + //TargetPhrase and source sentence (during pre-calculation) + //TODO: should the context also include some info on where the TargetPhrase + //is anchored (assuming it's lexicalised), which is available at pre-calc? + const ChartHypothesis* m_hypothesis; + const TargetPhrase& m_targetPhrase; + const InputType& m_source; + +public: + ChartBasedFeatureContext(const ChartHypothesis* hypothesis); + ChartBasedFeatureContext(const TargetPhrase& targetPhrase, + const InputType& source); + + const InputType& GetSource() const; + const TargetPhrase& GetTargetPhrase() const; + +}; + /** base class for all feature functions. * @todo is this for pb & hiero too? @@ -23,9 +78,12 @@ class FeatureFunction: public ScoreProducer { public: - virtual bool IsStateless() const = 0; + FeatureFunction(const std::string& description, size_t numScoreComponents) : + ScoreProducer(description, numScoreComponents) {} + virtual bool IsStateless() const = 0; virtual ~FeatureFunction(); - + + float GetSparseProducerWeight() const { return 1; } }; /** base class for all stateless feature functions. @@ -35,17 +93,28 @@ class StatelessFeatureFunction: public FeatureFunction { public: - //! Evaluate for stateless feature functions. Implement this. - virtual void Evaluate( - const TargetPhrase& cur_hypo, - ScoreComponentCollection* accumulator) const; - - // If true, this value is expected to be included in the - // ScoreBreakdown in the TranslationOption once it has been - // constructed. - // Default: true + StatelessFeatureFunction(const std::string& description, size_t numScoreComponents) : + FeatureFunction(description, numScoreComponents) {} + /** + * This should be implemented for features that apply to phrase-based models. + **/ + virtual void Evaluate(const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const = 0; + + /** + * Same for chart-based features. + **/ + virtual void EvaluateChart(const ChartBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const = 0; + + //If true, then the feature is evaluated before search begins, and stored in + //the TranslationOptionCollection. virtual bool ComputeValueInTranslationOption() const; + //!If true, the feature is stored in the ttable, so gets copied into the + //TargetPhrase and does not need cached in the TranslationOption + virtual bool ComputeValueInTranslationTable() const {return false;} + bool IsStateless() const; }; @@ -56,6 +125,8 @@ class StatefulFeatureFunction: public FeatureFunction { public: + StatefulFeatureFunction(const std::string& description, size_t numScoreComponents) : + FeatureFunction(description,numScoreComponents) {} /** * \brief This interface should be implemented. @@ -71,7 +142,7 @@ public: virtual FFState* EvaluateChart( const ChartHypothesis& /* cur_hypo */, - int /* featureID */, + int /* featureID - used to index the state in the previous hypotheses */, ScoreComponentCollection* accumulator) const = 0; //! return the state associated with the empty hypothesis for a given sentence diff --git a/moses/src/FeatureVector.cpp b/moses/src/FeatureVector.cpp new file mode 100644 index 000000000..43d43748e --- /dev/null +++ b/moses/src/FeatureVector.cpp @@ -0,0 +1,774 @@ +/* + Moses - factored phrase-based language decoder + Copyright (C) 2010 University of Edinburgh + + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + */ + +#include <algorithm> +#include <cmath> +#include <fstream> +#include <sstream> +#include <stdexcept> + +#include "FeatureVector.h" + +using namespace std; + + +namespace Moses { + + const string FName::SEP = "_"; + FName::Name2Id FName::name2id; + vector<string> FName::id2name; + FName::Id2Count FName::id2hopeCount; + FName::Id2Count FName::id2fearCount; +#ifdef WITH_THREADS + boost::shared_mutex FName::m_idLock; +#endif + + void FName::init(const string& name) { +#ifdef WITH_THREADS + //reader lock + boost::shared_lock<boost::shared_mutex> lock(m_idLock); +#endif + Name2Id::iterator i = name2id.find(name); + if (i != name2id.end()) { + m_id = i->second; + } else { +#ifdef WITH_THREADS + //release the reader lock, and upgrade to writer lock + lock.unlock(); + boost::upgrade_lock<boost::shared_mutex> upgradeLock(m_idLock); + boost::upgrade_to_unique_lock<boost::shared_mutex> uniqueLock(upgradeLock); +#endif + //Need to check again if the id is in the map, as someone may have added + //it while we were waiting on the writer lock. + if (i != name2id.end()) { + m_id = i->second; + } else { + m_id = name2id.size(); + name2id[name] = m_id; + id2name.push_back(name); + } + } + } + + size_t FName::getId(const string& name) { + Name2Id::iterator i = name2id.find(name); + assert (i != name2id.end()); + return i->second; + } + + size_t FName::getHopeIdCount(const string& name) { + Name2Id::iterator i = name2id.find(name); + if (i != name2id.end()) { + float id = i->second; + return id2hopeCount[id]; + } + return 0; + } + + size_t FName::getFearIdCount(const string& name) { + Name2Id::iterator i = name2id.find(name); + if (i != name2id.end()) { + float id = i->second; + return id2fearCount[id]; + } + return 0; + } + + void FName::incrementHopeId(const string& name) { + Name2Id::iterator i = name2id.find(name); + assert(i != name2id.end()); +#ifdef WITH_THREADS + // get upgradable lock and upgrade to writer lock + boost::upgrade_lock<boost::shared_mutex> upgradeLock(m_idLock); + boost::upgrade_to_unique_lock<boost::shared_mutex> uniqueLock(upgradeLock); +#endif + id2hopeCount[i->second] += 1; + } + + void FName::incrementFearId(const string& name) { + Name2Id::iterator i = name2id.find(name); + assert(i != name2id.end()); +#ifdef WITH_THREADS + // get upgradable lock and upgrade to writer lock + boost::upgrade_lock<boost::shared_mutex> upgradeLock(m_idLock); + boost::upgrade_to_unique_lock<boost::shared_mutex> uniqueLock(upgradeLock); +#endif + id2fearCount[i->second] += 1; + } + + void FName::eraseId(size_t id) { +#ifdef WITH_THREADS + // get upgradable lock and upgrade to writer lock + boost::upgrade_lock<boost::shared_mutex> upgradeLock(m_idLock); + boost::upgrade_to_unique_lock<boost::shared_mutex> uniqueLock(upgradeLock); +#endif + id2hopeCount.erase(id); + id2fearCount.erase(id); + } + + std::ostream& operator<<( std::ostream& out, const FName& name) { + out << name.name(); + return out; + } + + size_t FName::hash() const { + return boost::hash_value(m_id); + } + + const std::string& FName::name() const { + return id2name[m_id]; + } + + + bool FName::operator==(const FName& rhs) const { + return m_id == rhs.m_id; + } + + bool FName::operator!=(const FName& rhs) const { + return ! (*this == rhs); + } + + FVector::FVector(size_t coreFeatures) : m_coreFeatures(coreFeatures) {} + + void FVector::resize(size_t newsize) { + valarray<FValue> oldValues(m_coreFeatures); + m_coreFeatures.resize(newsize); + for (size_t i = 0; i < min(m_coreFeatures.size(), oldValues.size()); ++i) { + m_coreFeatures[i] = oldValues[i]; + } + } + + void FVector::clear() { + m_coreFeatures.resize(0); + m_features.clear(); + } + + bool FVector::load(const std::string& filename) { + clear(); + ifstream in (filename.c_str()); + if (!in) { + return false; + } + string line; + while(getline(in,line)) { + if (line[0] == '#') continue; + istringstream linestream(line); + string namestring; + FValue value; + linestream >> namestring; + linestream >> value; + FName fname(namestring); + //cerr << "Setting sparse weight " << fname << " to value " << value << "." << endl; + set(fname,value); + } + return true; + } + + void FVector::save(const string& filename) const { + ofstream out(filename.c_str()); + if (!out) { + ostringstream msg; + msg << "Unable to open " << filename; + throw runtime_error(msg.str()); + } + write(out); + out.close(); + } + + void FVector::write(ostream& out) const { + for (const_iterator i = cbegin(); i != cend(); ++i) { + out << i->first << " " << i->second << endl; + } + } + + static bool equalsTolerance(FValue lhs, FValue rhs) { + if (lhs == rhs) return true; + static const FValue TOLERANCE = 1e-4; + FValue diff = abs(lhs-rhs); + FValue mean = (abs(lhs)+abs(rhs))/2; + //cerr << "ET " << lhs << " " << rhs << " " << diff << " " << mean << " " << endl; + return diff/mean < TOLERANCE ; + } + + bool FVector::operator== (const FVector& rhs) const { + if (this == &rhs) { + return true; + } + if (m_coreFeatures.size() != rhs.m_coreFeatures.size()) { + return false; + } + for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + if (!equalsTolerance(m_coreFeatures[i], rhs.m_coreFeatures[i])) return false; + } + for (const_iterator i = cbegin(); i != cend(); ++i) { + if (!equalsTolerance(i->second,rhs.get(i->first))) return false; + } + for (const_iterator i = rhs.cbegin(); i != rhs.cend(); ++i) { + if (!equalsTolerance(i->second, get(i->first))) return false; + } + return true; + } + + bool FVector::operator!= (const FVector& rhs) const { + return ! (*this == rhs); + } + + ProxyFVector FVector::operator[](const FName& name) { + // At this point, we don't know whether operator[] was called, so we return + // a proxy object and defer the decision until later + return ProxyFVector(this, name); + } + + /** Equivalent for core features. */ + FValue& FVector::operator[](size_t index) { + return m_coreFeatures[index]; + } + + + FValue FVector::operator[](const FName& name) const { + return get(name); + } + + FValue FVector::operator[](size_t index) const { + return m_coreFeatures[index]; + } + + ostream& FVector::print(ostream& out) const { + out << "core=("; + for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + out << m_coreFeatures[i]; + if (i + 1 < m_coreFeatures.size()) { + out << ","; + } + } + out << ") "; + for (const_iterator i = cbegin(); i != cend(); ++i) { + if (i != cbegin()) + out << " "; + out << i->first << "=" << i->second; + } + return out; + } + + ostream& operator<<(ostream& out, const FVector& fv) { + return fv.print(out); + } + + const FValue& FVector::get(const FName& name) const { + static const FValue DEFAULT = 0; + const_iterator fi = m_features.find(name); + if (fi == m_features.end()) { + return DEFAULT; + } else { + return fi->second; + } + } + + const FValue& FVector::getBackoff(const FName& name, float backoff) const { + const_iterator fi = m_features.find(name); + if (fi == m_features.end()) { + return backoff; + } else { + return fi->second; + } + } + + void FVector::thresholdScale(FValue maxValue ) { + FValue factor = 1.0; + for (const_iterator i = cbegin(); i != cend(); ++i) { + FValue value = i->second; + if (abs(value)*factor > maxValue) { + factor = abs(value) / maxValue; + } + } + operator*=(factor); + } + + void FVector::capMax(FValue maxValue) { + for (const_iterator i = cbegin(); i != cend(); ++i) + if (i->second > maxValue) + set(i->first, maxValue); + } + + void FVector::capMin(FValue minValue) { + for (const_iterator i = cbegin(); i != cend(); ++i) + if (i->second < minValue) + set(i->first, minValue); + } + + void FVector::set(const FName& name, const FValue& value) { + m_features[name] = value; + } + + void FVector::printCoreFeatures() { + cerr << "core=("; + for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + cerr << m_coreFeatures[i]; + if (i + 1 < m_coreFeatures.size()) { + cerr << ","; + } + } + cerr << ") "; + } + + FVector& FVector::operator+= (const FVector& rhs) { + if (rhs.m_coreFeatures.size() > m_coreFeatures.size()) + resize(rhs.m_coreFeatures.size()); + for (const_iterator i = rhs.cbegin(); i != rhs.cend(); ++i) + set(i->first, get(i->first) + i->second); + for (size_t i = 0; i < rhs.m_coreFeatures.size(); ++i) + m_coreFeatures[i] += rhs.m_coreFeatures[i]; + return *this; + } + + // add only sparse features + void FVector::sparsePlusEquals(const FVector& rhs) { + for (const_iterator i = rhs.cbegin(); i != rhs.cend(); ++i) + set(i->first, get(i->first) + i->second); + } + + // assign only core features + void FVector::coreAssign(const FVector& rhs) { + for (size_t i = 0; i < rhs.m_coreFeatures.size(); ++i) + m_coreFeatures[i] = rhs.m_coreFeatures[i]; + } + + void FVector::incrementSparseHopeFeatures() { + for (const_iterator i = cbegin(); i != cend(); ++i) + FName::incrementHopeId((i->first).name()); + } + + void FVector::incrementSparseFearFeatures() { + for (const_iterator i = cbegin(); i != cend(); ++i) + FName::incrementFearId((i->first).name()); + } + + void FVector::printSparseHopeFeatureCounts(std::ofstream& out) { + for (const_iterator i = cbegin(); i != cend(); ++i) + out << (i->first).name() << ": " << FName::getHopeIdCount((i->first).name()) << std::endl; + } + + void FVector::printSparseFearFeatureCounts(std::ofstream& out) { + for (const_iterator i = cbegin(); i != cend(); ++i) + out << (i->first).name() << ": " << FName::getFearIdCount((i->first).name()) << std::endl; + } + + void FVector::printSparseHopeFeatureCounts() { + for (const_iterator i = cbegin(); i != cend(); ++i) + std::cerr << (i->first).name() << ": " << FName::getHopeIdCount((i->first).name()) << std::endl; + } + + void FVector::printSparseFearFeatureCounts() { + for (const_iterator i = cbegin(); i != cend(); ++i) + std::cerr << (i->first).name() << ": " << FName::getFearIdCount((i->first).name()) << std::endl; + } + + size_t FVector::pruneSparseFeatures(size_t threshold) { + size_t count = 0; + vector<FName> toErase; + for (const_iterator i = cbegin(); i != cend(); ++i) { + const std::string& fname = (i->first).name(); + if (FName::getHopeIdCount(fname) < threshold && FName::getFearIdCount(fname) < threshold) { + toErase.push_back(i->first); + std::cerr << "pruning: " << fname << " (" << FName::getHopeIdCount(fname) << ", " << FName::getFearIdCount(fname) << ")" << std::endl; + FName::eraseId(FName::getId(fname)); + ++count; + } + } + + for (size_t i = 0; i < toErase.size(); ++i) + m_features.erase(toErase[i]); + + return count; + } + + size_t FVector::pruneZeroWeightFeatures() { + size_t count = 0; + vector<FName> toErase; + for (const_iterator i = cbegin(); i != cend(); ++i) { + const std::string& fname = (i->first).name(); + if (i->second == 0) { + toErase.push_back(i->first); + //std::cerr << "prune: " << fname << std::endl; + FName::eraseId(FName::getId(fname)); + ++count; + } + } + + for (size_t i = 0; i < toErase.size(); ++i) + m_features.erase(toErase[i]); + + return count; + } + + void FVector::updateConfidenceCounts(const FVector& weightUpdate, bool signedCounts) { + for (size_t i = 0; i < weightUpdate.m_coreFeatures.size(); ++i) { + if (signedCounts) { + //int sign = weightUpdate.m_coreFeatures[i] >= 0 ? 1 : -1; + //m_coreFeatures[i] += (weightUpdate.m_coreFeatures[i] * weightUpdate.m_coreFeatures[i]) * sign; + m_coreFeatures[i] += weightUpdate.m_coreFeatures[i]; + } + else + //m_coreFeatures[i] += (weightUpdate.m_coreFeatures[i] * weightUpdate.m_coreFeatures[i]); + m_coreFeatures[i] += abs(weightUpdate.m_coreFeatures[i]); + } + + for (const_iterator i = weightUpdate.cbegin(); i != weightUpdate.cend(); ++i) { + if (weightUpdate[i->first] == 0) + continue; + float value = get(i->first); + if (signedCounts) { + //int sign = weightUpdate[i->first] >= 0 ? 1 : -1; + //value += (weightUpdate[i->first] * weightUpdate[i->first]) * sign; + value += weightUpdate[i->first]; + } + else + //value += (weightUpdate[i->first] * weightUpdate[i->first]); + value += abs(weightUpdate[i->first]); + set(i->first, value); + } + } + + void FVector::updateLearningRates(float decay_core, float decay_sparse, const FVector &confidenceCounts, float core_r0, float sparse_r0) { + for (size_t i = 0; i < confidenceCounts.m_coreFeatures.size(); ++i) { + m_coreFeatures[i] = 1.0/(1.0/core_r0 + decay_core * abs(confidenceCounts.m_coreFeatures[i])); + } + + for (const_iterator i = confidenceCounts.cbegin(); i != confidenceCounts.cend(); ++i) { + float value = 1.0/(1.0/sparse_r0 + decay_sparse * abs(i->second)); + set(i->first, value); + } + } + + // count non-zero occurrences for all sparse features + void FVector::setToBinaryOf(const FVector& rhs) { + for (const_iterator i = rhs.cbegin(); i != rhs.cend(); ++i) + if (rhs.get(i->first) != 0) + set(i->first, 1); + for (size_t i = 0; i < rhs.m_coreFeatures.size(); ++i) + m_coreFeatures[i] = 1; + } + + // divide only core features by scalar + FVector& FVector::coreDivideEquals(float scalar) { + for (size_t i = 0; i < m_coreFeatures.size(); ++i) + m_coreFeatures[i] /= scalar; + return *this; + } + + // lhs vector is a sum of vectors, rhs vector holds number of non-zero summands + FVector& FVector::divideEquals(const FVector& rhs) { + assert(m_coreFeatures.size() == rhs.m_coreFeatures.size()); + for (const_iterator i = rhs.cbegin(); i != rhs.cend(); ++i) + set(i->first, get(i->first)/rhs.get(i->first)); // divide by number of summands + for (size_t i = 0; i < rhs.m_coreFeatures.size(); ++i) + m_coreFeatures[i] /= rhs.m_coreFeatures[i]; // divide by number of summands + return *this; + } + + FVector& FVector::operator-= (const FVector& rhs) { + if (rhs.m_coreFeatures.size() > m_coreFeatures.size()) + resize(rhs.m_coreFeatures.size()); + for (const_iterator i = rhs.cbegin(); i != rhs.cend(); ++i) + set(i->first, get(i->first) -(i->second)); + for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + if (i < rhs.m_coreFeatures.size()) { + m_coreFeatures[i] -= rhs.m_coreFeatures[i]; + } + } + return *this; + } + + FVector& FVector::operator*= (const FVector& rhs) { + if (rhs.m_coreFeatures.size() > m_coreFeatures.size()) { + resize(rhs.m_coreFeatures.size()); + } + for (iterator i = begin(); i != end(); ++i) { + FValue lhsValue = i->second; + FValue rhsValue = rhs.get(i->first); + set(i->first,lhsValue*rhsValue); + } + for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + if (i < rhs.m_coreFeatures.size()) { + m_coreFeatures[i] *= rhs.m_coreFeatures[i]; + } else { + m_coreFeatures[i] = 0; + } + } + return *this; + } + + FVector& FVector::operator/= (const FVector& rhs) { + if (rhs.m_coreFeatures.size() > m_coreFeatures.size()) { + resize(rhs.m_coreFeatures.size()); + } + for (iterator i = begin(); i != end(); ++i) { + FValue lhsValue = i->second; + FValue rhsValue = rhs.get(i->first); + set(i->first, lhsValue / rhsValue) ; + } + for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + if (i < rhs.m_coreFeatures.size()) { + m_coreFeatures[i] /= rhs.m_coreFeatures[i]; + } else { + if (m_coreFeatures[i] < 0) { + m_coreFeatures[i] = -numeric_limits<FValue>::infinity(); + } else if (m_coreFeatures[i] > 0) { + m_coreFeatures[i] = numeric_limits<FValue>::infinity(); + } + } + } + return *this; + } + + FVector& FVector::operator*= (const FValue& rhs) { + //NB Could do this with boost::bind ? + for (iterator i = begin(); i != end(); ++i) { + i->second *= rhs; + } + m_coreFeatures *= rhs; + return *this; + } + + FVector& FVector::operator/= (const FValue& rhs) { + for (iterator i = begin(); i != end(); ++i) { + i->second /= rhs; + } + m_coreFeatures /= rhs; + return *this; + } + + FVector& FVector::multiplyEqualsBackoff(const FVector& rhs, float backoff) { + if (rhs.m_coreFeatures.size() > m_coreFeatures.size()) { + resize(rhs.m_coreFeatures.size()); + } + for (iterator i = begin(); i != end(); ++i) { + FValue lhsValue = i->second; + FValue rhsValue = rhs.getBackoff(i->first, backoff); + set(i->first,lhsValue*rhsValue); + } + for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + if (i < rhs.m_coreFeatures.size()) { + m_coreFeatures[i] *= rhs.m_coreFeatures[i]; + } else { + m_coreFeatures[i] = 0; + } + } + return *this; + } + + FVector& FVector::multiplyEquals(float core_r0, float sparse_r0) { + for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + m_coreFeatures[i] *= core_r0; + } + for (iterator i = begin(); i != end(); ++i) + set(i->first,(i->second)*sparse_r0); + return *this; + } + + FValue FVector::l1norm() const { + FValue norm = 0; + for (const_iterator i = cbegin(); i != cend(); ++i) { + norm += abs(i->second); + } + for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + norm += abs(m_coreFeatures[i]); + } + return norm; + } + + FValue FVector::l1norm_coreFeatures() const { + FValue norm = 0; + // ignore Bleu score feature (last feature) + for (size_t i = 0; i < m_coreFeatures.size()-1; ++i) + norm += abs(m_coreFeatures[i]); + return norm; + } + + FValue FVector::l2norm() const { + return sqrt(inner_product(*this)); + } + + FValue FVector::linfnorm() const { + FValue norm = 0; + for (const_iterator i = cbegin(); i != cend(); ++i) { + float absValue = abs(i->second); + if (absValue > norm) + norm = absValue; + } + for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + float absValue = abs(m_coreFeatures[i]); + if (absValue > norm) + norm = absValue; + } + return norm; + } + + size_t FVector::l1regularize(float lambda) { + for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + float value = m_coreFeatures[i]; + if (value > 0) { + m_coreFeatures[i] = max(0.0f, value - lambda); + } + else { + m_coreFeatures[i] = min(0.0f, value + lambda); + } + } + + size_t numberPruned = size(); + vector<FName> toErase; + for (iterator i = begin(); i != end(); ++i) { + float value = i->second; + if (value != 0.0f) { + if (value > 0) + value = max(0.0f, value - lambda); + else + value = min(0.0f, value + lambda); + + if (value != 0.0f) + i->second = value; + else { + toErase.push_back(i->first); + const std::string& fname = (i->first).name(); + FName::eraseId(FName::getId(fname)); + } + } + } + + // erase features that have become zero + for (size_t i = 0; i < toErase.size(); ++i) + m_features.erase(toErase[i]); + numberPruned -= size(); + return numberPruned; + } + + void FVector::l2regularize(float lambda) { + for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + m_coreFeatures[i] *= (1 - lambda); + } + + for (iterator i = begin(); i != end(); ++i) { + i->second *= (1 - lambda); + } + } + + size_t FVector::sparseL1regularize(float lambda) { + /*for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + float value = m_coreFeatures[i]; + if (value > 0) { + m_coreFeatures[i] = max(0.0f, value - lambda); + } + else { + m_coreFeatures[i] = min(0.0f, value + lambda); + } + }*/ + + size_t numberPruned = size(); + vector<FName> toErase; + for (iterator i = begin(); i != end(); ++i) { + float value = i->second; + if (value != 0.0f) { + if (value > 0) + value = max(0.0f, value - lambda); + else + value = min(0.0f, value + lambda); + + if (value != 0.0f) + i->second = value; + else { + toErase.push_back(i->first); + const std::string& fname = (i->first).name(); + FName::eraseId(FName::getId(fname)); + } + } + } + + // erase features that have become zero + for (size_t i = 0; i < toErase.size(); ++i) + m_features.erase(toErase[i]); + numberPruned -= size(); + return numberPruned; + } + + void FVector::sparseL2regularize(float lambda) { + /*for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + m_coreFeatures[i] *= (1 - lambda); + }*/ + + for (iterator i = begin(); i != end(); ++i) { + i->second *= (1 - lambda); + } + } + + FValue FVector::sum() const { + FValue sum = 0; + for (const_iterator i = cbegin(); i != cend(); ++i) { + sum += i->second; + } + sum += m_coreFeatures.sum(); + return sum; + } + + FValue FVector::inner_product(const FVector& rhs) const { + CHECK(m_coreFeatures.size() == rhs.m_coreFeatures.size()); + FValue product = 0.0; + for (const_iterator i = cbegin(); i != cend(); ++i) { + product += ((i->second)*(rhs.get(i->first))); + } + for (size_t i = 0; i < m_coreFeatures.size(); ++i) { + product += m_coreFeatures[i]*rhs.m_coreFeatures[i]; + } + return product; + } + + const FVector operator+(const FVector& lhs, const FVector& rhs) { + return FVector(lhs) += rhs; + } + + const FVector operator-(const FVector& lhs, const FVector& rhs) { + return FVector(lhs) -= rhs; + } + + const FVector operator*(const FVector& lhs, const FVector& rhs) { + return FVector(lhs) *= rhs; + } + + const FVector operator/(const FVector& lhs, const FVector& rhs) { + return FVector(lhs) /= rhs; + } + + + const FVector operator*(const FVector& lhs, const FValue& rhs) { + return FVector(lhs) *= rhs; + } + + const FVector operator/(const FVector& lhs, const FValue& rhs) { + return FVector(lhs) /= rhs; + } + + FValue inner_product(const FVector& lhs, const FVector& rhs) { + if (lhs.size() >= rhs.size()) { + return rhs.inner_product(lhs); + } else { + return lhs.inner_product(rhs); + } + } +} diff --git a/moses/src/FeatureVector.h b/moses/src/FeatureVector.h new file mode 100644 index 000000000..22c4a6a16 --- /dev/null +++ b/moses/src/FeatureVector.h @@ -0,0 +1,356 @@ +/* + Moses - factored phrase-based language decoder + Copyright (C) 2010 University of Edinburgh + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + */ +#pragma once + +#ifndef FEATUREVECTOR_H +#define FEATUREVECTOR_H + +#include <iostream> +#include <map> +#include <sstream> +#include <string> +#include <valarray> +#include <vector> + +#include <boost/functional/hash.hpp> +#include <boost/unordered_map.hpp> + +#ifdef MPI_ENABLE +#include <boost/serialization/access.hpp> +#include <boost/serialization/split_member.hpp> +#include <boost/serialization/string.hpp> +#include <boost/serialization/vector.hpp> +#include <boost/serialization/valarray.hpp> +#endif + +#ifdef WITH_THREADS +#include <boost/thread/shared_mutex.hpp> +#endif + +#include "util/check.hh" + +namespace Moses { + + typedef float FValue; + + /** + * Feature name + **/ + struct FName { + + static const std::string SEP; + + typedef boost::unordered_map<std::string,size_t> Name2Id; + typedef boost::unordered_map<size_t,size_t> Id2Count; + //typedef std::map<std::string, size_t> Name2Id; + static Name2Id name2id; + static std::vector<std::string> id2name; + static Id2Count id2hopeCount; + static Id2Count id2fearCount; + + //A feature name can either be initialised as a pair of strings, + //which will be concatenated with a SEP between them, or as + //a single string, which will be used as-is. + explicit FName(const std::string root, const std::string name) + {init(root + SEP + name);} + explicit FName(const std::string& name) + {init(name);} + + const std::string& name() const; + //const std::string& root() const {return m_root;} + + size_t hash() const; + + bool operator==(const FName& rhs) const ; + bool operator!=(const FName& rhs) const ; + + static size_t getId(const std::string& name); + static size_t getHopeIdCount(const std::string& name); + static size_t getFearIdCount(const std::string& name); + static void incrementHopeId(const std::string& name); + static void incrementFearId(const std::string& name); + static void eraseId(size_t id); + + private: + void init(const std::string& name); + size_t m_id; +#ifdef WITH_THREADS + //reader-writer lock + static boost::shared_mutex m_idLock; +#endif + }; + + std::ostream& operator<<(std::ostream& out,const FName& name); + + struct FNameEquals { + inline bool operator() (const FName& lhs, const FName& rhs) const { + return (lhs == rhs); + } + }; + + struct FNameHash + : std::unary_function<FName, std::size_t> + { + std::size_t operator()(const FName& x) const + { + return x.hash(); + } + }; + + class ProxyFVector; + + /** + * A sparse feature (or weight) vector. + **/ + class FVector + { + public: + /** Empty feature vector */ + FVector(size_t coreFeatures = 0); + + /* + * Change the number of core features + **/ + void resize(size_t newsize); + + typedef boost::unordered_map<FName,FValue,FNameHash, FNameEquals> FNVmap; + /** Iterators */ + typedef FNVmap::iterator iterator; + typedef FNVmap::const_iterator const_iterator; + iterator begin() {return m_features.begin();} + iterator end() {return m_features.end();} + const_iterator cbegin() const {return m_features.cbegin();} + const_iterator cend() const {return m_features.cend();} + + bool hasNonDefaultValue(FName name) const { return m_features.find(name) != m_features.end();} + void clear(); + + + /** Load from file - each line should be 'root[_name] value' */ + bool load(const std::string& filename); + void save(const std::string& filename) const; + void write(std::ostream& out) const ; + + /** Element access */ + ProxyFVector operator[](const FName& name); + FValue& operator[](size_t index); + FValue operator[](const FName& name) const; + FValue operator[](size_t index) const; + + /** Size */ + size_t size() const { + return m_features.size() + m_coreFeatures.size(); + } + + size_t coreSize() const { + return m_coreFeatures.size(); + } + + /** Equality */ + bool operator== (const FVector& rhs) const; + bool operator!= (const FVector& rhs) const; + + FValue inner_product(const FVector& rhs) const; + + friend class ProxyFVector; + + /**arithmetic */ + //Element-wise + //If one side has fewer core features, take the missing ones to be 0. + FVector& operator+= (const FVector& rhs); + FVector& operator-= (const FVector& rhs); + FVector& operator*= (const FVector& rhs); + FVector& operator/= (const FVector& rhs); + //Scalar + FVector& operator*= (const FValue& rhs); + FVector& operator/= (const FValue& rhs); + + FVector& multiplyEqualsBackoff(const FVector& rhs, float backoff); + FVector& multiplyEquals(float core_r0, float sparse_r0); + + FVector& max_equals(const FVector& rhs); + + /** norms and sums */ + FValue l1norm() const; + FValue l1norm_coreFeatures() const; + FValue l2norm() const; + FValue linfnorm() const; + size_t l1regularize(float lambda); + void l2regularize(float lambda); + size_t sparseL1regularize(float lambda); + void sparseL2regularize(float lambda); + FValue sum() const; + + /** pretty printing */ + std::ostream& print(std::ostream& out) const; + + /** additional */ + void printCoreFeatures(); + //scale so that abs. value is less than maxvalue + void thresholdScale(float maxValue ); + + void capMax(FValue maxValue); + void capMin(FValue minValue); + + void sparsePlusEquals(const FVector& rhs); + void coreAssign(const FVector& rhs); + + void incrementSparseHopeFeatures(); + void incrementSparseFearFeatures(); + void printSparseHopeFeatureCounts(std::ofstream& out); + void printSparseFearFeatureCounts(std::ofstream& out); + void printSparseHopeFeatureCounts(); + void printSparseFearFeatureCounts(); + size_t pruneSparseFeatures(size_t threshold); + size_t pruneZeroWeightFeatures(); + void updateConfidenceCounts(const FVector& weightUpdate, bool signedCounts); + void updateLearningRates(float decay_core, float decay_sparse, const FVector& confidence_counts, float core_r0, float sparse_r0); + + // vector which, for each element of the original vector, reflects whether an element is zero or non-zero + void setToBinaryOf(const FVector& rhs); + + // divide only core features by scalar + FVector& coreDivideEquals(float scalar); + + // divide each element by the number given in the rhs vector + FVector& divideEquals(const FVector& rhs); + +#ifdef MPI_ENABLE + friend class boost::serialization::access; +#endif + + private: + + /** Internal get and set. */ + const FValue& get(const FName& name) const; + const FValue& getBackoff(const FName& name, float backoff) const; + void set(const FName& name, const FValue& value); + + FNVmap m_features; + std::valarray<FValue> m_coreFeatures; + +#ifdef MPI_ENABLE + //serialization + template<class Archive> + void save(Archive &ar, const unsigned int version) const { + std::vector<std::string> names; + std::vector<FValue> values; + for (const_iterator i = cbegin(); i != cend(); ++i) { + std::ostringstream ostr; + ostr << i->first; + names.push_back(ostr.str()); + values.push_back(i->second); + } + ar << names; + ar << values; + ar << m_coreFeatures; + } + + template<class Archive> + void load(Archive &ar, const unsigned int version) { + clear(); + std::vector<std::string> names; + std::vector<FValue> values; + ar >> names; + ar >> values; + ar >> m_coreFeatures; + CHECK(names.size() == values.size()); + for (size_t i = 0; i < names.size(); ++i) { + set(FName(names[i]), values[i]); + } + } + + BOOST_SERIALIZATION_SPLIT_MEMBER() + +#endif + + }; + + std::ostream& operator<<( std::ostream& out, const FVector& fv); + //Element-wise operations + const FVector operator+(const FVector& lhs, const FVector& rhs); + const FVector operator-(const FVector& lhs, const FVector& rhs); + const FVector operator*(const FVector& lhs, const FVector& rhs); + const FVector operator/(const FVector& lhs, const FVector& rhs); + + //Scalar operations + const FVector operator*(const FVector& lhs, const FValue& rhs); + const FVector operator/(const FVector& lhs, const FValue& rhs); + + const FVector fvmax(const FVector& lhs, const FVector& rhs); + + FValue inner_product(const FVector& lhs, const FVector& rhs); + + struct FVectorPlus { + FVector operator()(const FVector& lhs, const FVector& rhs) const { + return lhs + rhs; + } + }; + + /** + * Used to help with subscript operator overloading. + * See http://stackoverflow.com/questions/1386075/overloading-operator-for-a-sparse-vector + **/ + class ProxyFVector { + public: + ProxyFVector(FVector *fv, const FName& name ) : m_fv(fv), m_name(name) {} + ProxyFVector &operator=(const FValue& value) { + // If we get here, we know that operator[] was called to perform a write access, + // so we can insert an item in the vector if needed + //std::cerr << "Inserting " << value << " into " << m_name << std::endl; + m_fv->set(m_name,value); + return *this; + + } + + operator FValue() { + // If we get here, we know that operator[] was called to perform a read access, + // so we can simply return the value from the vector + return m_fv->get(m_name); + } + + /*operator FValue&() { + return m_fv->m_features[m_name]; + }*/ + + FValue operator++() { + return ++m_fv->m_features[m_name]; + } + + FValue operator +=(FValue lhs) { + return (m_fv->m_features[m_name] += lhs); + } + + FValue operator -=(FValue lhs) { + return (m_fv->m_features[m_name] -= lhs); + } + + private: + FValue m_tmp; + + private: + FVector* m_fv; + const FName& m_name; + + }; + +} + +#endif diff --git a/moses/src/FeatureVectorTest.cpp b/moses/src/FeatureVectorTest.cpp new file mode 100644 index 000000000..af1829e62 --- /dev/null +++ b/moses/src/FeatureVectorTest.cpp @@ -0,0 +1,273 @@ +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2010- University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#include <boost/test/unit_test.hpp> + +#include "FeatureVector.h" + +using namespace Moses; +using namespace std; + +static const float TOL = 0.00001; + +BOOST_AUTO_TEST_SUITE(fv) + +BOOST_AUTO_TEST_CASE(vector_sum_diff) +{ + FVector f1,f2,f3; + FName n1("a"); + FName n2("b"); + FName n3("c"); + FName n4("d"); + f1[n1] = 1.2; f1[n2] = 1.4; f1[n3] = -0.1; + f2[n1] = 0.01; f2[n3] = 5.6; f2[n4] = 0.6; + f3[n1] =1.2; + FVector sum = f1 + f2; + FVector diff = f1 - f2; + BOOST_CHECK_CLOSE((FValue)sum[n1], 1.21, TOL); + BOOST_CHECK_CLOSE((FValue)sum[n2], 1.4, TOL); + BOOST_CHECK_CLOSE((FValue)sum[n3], 5.5, TOL); + BOOST_CHECK_CLOSE((FValue)sum[n4], 0.6, TOL); + BOOST_CHECK_CLOSE((FValue)diff[n1], 1.19, TOL); + BOOST_CHECK_CLOSE((FValue)diff[n2], 1.4, TOL); + BOOST_CHECK_CLOSE((FValue)diff[n3], -5.7, TOL); + BOOST_CHECK_CLOSE((FValue)diff[n4], -0.6, TOL); + f1 -= f3; + cerr << f1 << endl << f3 << endl ; + BOOST_CHECK_CLOSE((FValue)f1[n1],0,TOL); +} + + +BOOST_AUTO_TEST_CASE(scalar) +{ + FVector f1,f2; + FName n1("a"); + FName n2("b"); + FName n3("c"); + FName n4("d"); + f1[n1] = 0.2; f1[n2] = 9.178; f1[n3] = -0.1; + f2[n1] = 0.01; f2[n3] = 5.6; f2[n4] = 0.6; + FVector prod1 = f1 * 2; + FVector prod2 = f1 * -0.1; + FVector quot = f2 / 2; + BOOST_CHECK_CLOSE((FValue)prod1[n1], 0.4, TOL); + BOOST_CHECK_CLOSE((FValue)prod1[n2], 18.356, TOL); + BOOST_CHECK_CLOSE((FValue)prod1[n3], -0.2, TOL); + + BOOST_CHECK_CLOSE((FValue)prod2[n1], -0.02, TOL); + BOOST_CHECK_CLOSE((FValue)prod2[n2], -0.9178, TOL); + BOOST_CHECK_CLOSE((FValue)prod2[n3], 0.01, TOL); + + BOOST_CHECK_CLOSE((FValue)quot[n1], 0.005, TOL); + BOOST_CHECK_CLOSE((FValue)quot[n2], 0, TOL); + BOOST_CHECK_CLOSE((FValue)quot[n3], 2.8, TOL); + BOOST_CHECK_CLOSE((FValue)quot[n4], 0.3, TOL); +} + +BOOST_AUTO_TEST_CASE(inc) +{ + FVector f1; + FName n1("a"); + FName n2("b"); + f1[n1] = 2.3; f1[n2] = -0.4; + f1[n1]+=2; + BOOST_CHECK_CLOSE((FValue)f1[n1], 4.3, TOL); + BOOST_CHECK_CLOSE((FValue)f1[n2], -0.4, TOL); + + FValue res = ++f1[n2]; + BOOST_CHECK_CLOSE(res,0.6, TOL); + BOOST_CHECK_CLOSE((FValue)f1[n1], 4.3, TOL); + BOOST_CHECK_CLOSE((FValue)f1[n2], 0.6, TOL); +} + +BOOST_AUTO_TEST_CASE(vector_mult) +{ + FVector f1,f2; + FName n1("a"); + FName n2("b"); + FName n3("c"); + FName n4("d"); + f1[n1] = 0.2; f1[n2] = 9.178; f1[n3] = -0.1; + f2[n1] = 0.01; f2[n2] = 5.6; f2[n3] = 1; f2[n4] = 0.6; + FVector prod = f1 * f2; + FVector quot = f1/f2; + BOOST_CHECK_CLOSE((FValue)prod[n1], 0.002, TOL); + BOOST_CHECK_CLOSE((FValue)prod[n2], 51.3968, TOL); + BOOST_CHECK_CLOSE((FValue)prod[n3], -0.1, TOL); + BOOST_CHECK_CLOSE((FValue)prod[n4], 0, TOL); + + BOOST_CHECK_CLOSE((FValue)quot[n1], 20, TOL); + BOOST_CHECK_CLOSE((FValue)quot[n2], 1.63892865, TOL); + BOOST_CHECK_CLOSE((FValue)quot[n3], -0.1, TOL); + BOOST_CHECK_CLOSE((FValue)quot[n4], 0, TOL); +} + +BOOST_AUTO_TEST_CASE(core) +{ + FVector f1(2); + f1[0] = 1.3; + f1[1] = -1.9; + BOOST_CHECK_CLOSE(f1[0],1.3,TOL); + BOOST_CHECK_CLOSE(f1[1],-1.9,TOL); + f1[1] = 0.1; + BOOST_CHECK_CLOSE(f1[1],0.1,TOL); + + BOOST_CHECK_EQUAL(f1.size(),2); + + f1[FName("a")] = 1.2; + BOOST_CHECK_EQUAL(f1.size(),3); +} + +BOOST_AUTO_TEST_CASE(core_arith) +{ + FVector f1(2); + FVector f2(2); + FName n1("a"); + FName n2("b"); + f1[0] = 1.1; f1[1] = 0.25; f1[n1] = 3.6; f1[n2] = -1.5; + f2[0] = 0.5; f2[1] = -0.1; f2[n1] = 1; + + //vector ops + FVector sum = f1+f2; + FVector diff = f1-f2; + FVector prod = f1*f2; + FVector quot = f2/f1; + + BOOST_CHECK_CLOSE((FValue)sum[0], 1.6 , TOL); + BOOST_CHECK_CLOSE((FValue)sum[1], 0.15 , TOL); + BOOST_CHECK_CLOSE((FValue)sum[n1], 4.6 , TOL); + BOOST_CHECK_CLOSE((FValue)sum[n2], -1.5 , TOL); + + BOOST_CHECK_CLOSE((FValue)diff[0], 0.6 , TOL); + BOOST_CHECK_CLOSE((FValue)diff[1], 0.35 , TOL); + BOOST_CHECK_CLOSE((FValue)diff[n1], 2.6 , TOL); + BOOST_CHECK_CLOSE((FValue)diff[n2], -1.5 , TOL); + + BOOST_CHECK_CLOSE((FValue)prod[0], 0.55 , TOL); + BOOST_CHECK_CLOSE((FValue)prod[1], -0.025 , TOL); + BOOST_CHECK_CLOSE((FValue)prod[n1], 3.6 , TOL); + BOOST_CHECK_CLOSE((FValue)prod[n2], 0 , TOL); + + BOOST_CHECK_CLOSE((FValue)quot[0], 0.4545454545 , TOL); + BOOST_CHECK_CLOSE((FValue)quot[1], -0.4 , TOL); + BOOST_CHECK_CLOSE((FValue)quot[n1], 0.277777777 , TOL); + BOOST_CHECK_CLOSE((FValue)quot[n2], 0 , TOL); + + //with different length vectors + FVector f3(2); + FVector f4(1); + f3[0] = 2; f3[1] = -1; + f4[0] = 5; + + FVector sum1 = f3 + f4; + FVector sum2 = f4 + f3; + BOOST_CHECK_EQUAL(sum1,sum2); + BOOST_CHECK_CLOSE(sum1[0], 7, TOL); + BOOST_CHECK_CLOSE(sum1[1], -1, TOL); + + FVector diff1 = f3 - f4; + FVector diff2 = f4 - f3; + BOOST_CHECK_CLOSE(diff1[0], -3, TOL); + BOOST_CHECK_CLOSE(diff1[1], -1, TOL); + BOOST_CHECK_CLOSE(diff2[0], 3, TOL); + BOOST_CHECK_CLOSE(diff2[1], 1, TOL); + + FVector prod1 = f3 * f4; + FVector prod2 = f4 * f3; + BOOST_CHECK_EQUAL(prod1,prod2); + BOOST_CHECK_CLOSE(prod1[0], 10, TOL); + BOOST_CHECK_CLOSE(prod1[1], 0, TOL); + + FVector quot1 = f3 / f4; + FVector quot2 = f4 / f3; + BOOST_CHECK_CLOSE(quot1[0], 0.4, TOL); + BOOST_CHECK_EQUAL(quot1[1], -numeric_limits<float>::infinity()); + BOOST_CHECK_CLOSE(quot2[0], 2.5, TOL); + BOOST_CHECK_CLOSE(quot2[1], 0, TOL); + +} + +BOOST_AUTO_TEST_CASE(core_scalar) +{ + FVector f1(3); + FName n1("a"); + f1[0] = 1.5; f1[1] = 2.1; f1[2] = 4; f1[n1] = -0.5; + + FVector prod = f1*2; + FVector quot = f1/5; + + BOOST_CHECK_CLOSE((FValue)prod[0], 3 , TOL); + BOOST_CHECK_CLOSE((FValue)prod[1], 4.2 , TOL); + BOOST_CHECK_CLOSE((FValue)prod[2], 8 , TOL); + BOOST_CHECK_CLOSE((FValue)prod[n1],-1 , TOL); + + BOOST_CHECK_CLOSE((FValue)quot[0], 0.3 , TOL); + BOOST_CHECK_CLOSE((FValue)quot[1], 0.42 , TOL); + BOOST_CHECK_CLOSE((FValue)quot[2], 0.8 , TOL); + BOOST_CHECK_CLOSE((FValue)quot[n1],-0.1 , TOL); + +} + +BOOST_AUTO_TEST_CASE(l1norm) +{ + FVector f1(3); + FName n1("a"); + f1[0] = 1.5; f1[1] = 2.1; f1[2] = 4; f1[n1] = -0.5; + FValue n = f1.l1norm(); + BOOST_CHECK_CLOSE((FValue)n, abs(1.5)+abs(2.1)+abs(4)+abs(-0.5), TOL); +} + + +BOOST_AUTO_TEST_CASE(sum) +{ + FVector f1(3); + FName n1("a"); + FName n2("b"); + f1[0] = 1.5; f1[1] = 2.1; f1[2] = 4; f1[n1] = -0.5; f1[n2] = 2.7; + FValue n = f1.sum(); + BOOST_CHECK_CLOSE((FValue)n, 1.5+2.1+4-0.5+2.7, TOL); +} + +BOOST_AUTO_TEST_CASE(l2norm) +{ + FVector f1(3); + FName n1("a"); + f1[0] = 1.5; f1[1] = 2.1; f1[2] = 4; f1[n1] = -0.5; + FValue n = f1.l2norm(); + BOOST_CHECK_CLOSE((FValue)n, sqrt((1.5*1.5)+(2.1*2.1)+(4*4)+(-0.5*-0.5)), TOL); +} + +BOOST_AUTO_TEST_CASE(ip) +{ + FVector f1(2); + FVector f2(2); + FName n1("a"); + FName n2("b"); + FName n3("c"); + f1[0] = 1.1; f1[1] = -0.1; ; f1[n2] = -1.5; f1[n3] = 2.2; + f2[0] = 0.5; f2[1] = 0.25; f2[n1] = 1; f2[n3] = 2.4; + FValue p1 = inner_product(f1,f2); + FValue p2 = inner_product(f2,f1); + BOOST_CHECK_CLOSE(p1,p2,TOL); + BOOST_CHECK_CLOSE((FValue)p1, 1.1*0.5 + -0.1*0.25 + 2.2*2.4, TOL); +} + + +BOOST_AUTO_TEST_SUITE_END() + diff --git a/moses/src/GenerationDictionary.cpp b/moses/src/GenerationDictionary.cpp index 8540d2707..f7b35fb63 100644 --- a/moses/src/GenerationDictionary.cpp +++ b/moses/src/GenerationDictionary.cpp @@ -33,13 +33,10 @@ using namespace std; namespace Moses { -GenerationDictionary::GenerationDictionary(size_t numFeatures, ScoreIndexManager &scoreIndexManager, - const std::vector<FactorType> &input, - const std::vector<FactorType> &output) - : Dictionary(numFeatures), DecodeFeature(input,output) -{ - scoreIndexManager.AddScoreProducer(this); -} + GenerationDictionary::GenerationDictionary(size_t numFeatures, + const std::vector<FactorType> &input, + const std::vector<FactorType> &output) + : Dictionary(numFeatures), DecodeFeature("Generation",numFeatures,input,output) {} bool GenerationDictionary::Load(const std::string &filePath, FactorDirection direction) { @@ -118,22 +115,6 @@ GenerationDictionary::~GenerationDictionary() } } -size_t GenerationDictionary::GetNumScoreComponents() const -{ - return m_numScoreComponent; -} - -std::string GenerationDictionary::GetScoreProducerDescription(unsigned) const -{ - return "GenerationScore,file=" + m_filePath; -} - -std::string GenerationDictionary::GetScoreProducerWeightShortName(unsigned) const -{ - return "g"; -} - - const OutputWordCollection *GenerationDictionary::FindWord(const Word &word) const { const OutputWordCollection *ret; diff --git a/moses/src/GenerationDictionary.h b/moses/src/GenerationDictionary.h index a666ae4af..af7765b35 100644 --- a/moses/src/GenerationDictionary.h +++ b/moses/src/GenerationDictionary.h @@ -24,6 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include <list> #include <map> +#include <stdexcept> #include <vector> #include "ScoreComponentCollection.h" #include "Phrase.h" @@ -56,35 +57,53 @@ public: * \param numFeatures number of score components, as specified in ini file */ GenerationDictionary( - size_t numFeatures, - ScoreIndexManager &scoreIndexManager, - const std::vector<FactorType> &input, - const std::vector<FactorType> &output); - virtual ~GenerationDictionary(); - - // returns Generate - DecodeType GetDecodeType() const { - return Generate; + size_t numFeatures, + const std::vector<FactorType> &input, + const std::vector<FactorType> &output); + virtual ~GenerationDictionary(); + + // returns Generate + DecodeType GetDecodeType() const + { + return Generate; + } + + //! load data file + bool Load(const std::string &filePath, FactorDirection direction); + + std::string GetScoreProducerWeightShortName(unsigned) const + { + return "g"; + } + + /** number of unique input entries in the generation table. + * NOT the number of lines in the generation table + */ + size_t GetSize() const + { + return m_collection.size(); + } + /** returns a bag of output words, OutputWordCollection, for a particular input word. + * Or NULL if the input word isn't found. The search function used is the WordComparer functor + */ + const OutputWordCollection *FindWord(const Word &word) const; + virtual bool ComputeValueInTranslationOption() const; + + //Usual feature function methods are not implemented + virtual void Evaluate(const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const + { + throw std::logic_error("GenerationDictionary::Evaluate() Not implemented"); } - //! load data file - bool Load(const std::string &filePath, FactorDirection direction); + virtual void EvaluateChart(const ChartBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const + { + throw std::logic_error("GenerationDictionary.Evaluate() Not implemented"); + } - size_t GetNumScoreComponents() const; - std::string GetScoreProducerDescription(unsigned) const; - std::string GetScoreProducerWeightShortName(unsigned) const; + virtual bool ComputeValueInTranslationTable() const {return true;} - /** number of unique input entries in the generation table. - * NOT the number of lines in the generation table - */ - size_t GetSize() const { - return m_collection.size(); - } - /** returns a bag of output words, OutputWordCollection, for a particular input word. - * Or NULL if the input word isn't found. The search function used is the WordComparer functor - */ - const OutputWordCollection *FindWord(const Word &word) const; - virtual bool ComputeValueInTranslationOption() const; }; diff --git a/moses/src/GlobalLexicalModel.cpp b/moses/src/GlobalLexicalModel.cpp index 84b3a3838..474ca8e8f 100644 --- a/moses/src/GlobalLexicalModel.cpp +++ b/moses/src/GlobalLexicalModel.cpp @@ -2,6 +2,7 @@ #include "GlobalLexicalModel.h" #include "StaticData.h" #include "InputFileStream.h" +#include "TranslationOption.h" #include "UserMessage.h" using namespace std; @@ -9,18 +10,12 @@ using namespace std; namespace Moses { GlobalLexicalModel::GlobalLexicalModel(const string &filePath, - const float weight, const vector< FactorType >& inFactors, const vector< FactorType >& outFactors) + : StatelessFeatureFunction("GlobalLexicalModel",1) { std::cerr << "Creating global lexical model...\n"; - // register as score producer - const_cast<ScoreIndexManager&>(StaticData::Instance().GetScoreIndexManager()).AddScoreProducer(this); - std::vector< float > weights; - weights.push_back( weight ); - const_cast<StaticData&>(StaticData::Instance()).SetWeightsForScoreProducer(this, weights); - // load model LoadData( filePath, inFactors, outFactors ); @@ -163,9 +158,11 @@ float GlobalLexicalModel::GetFromCacheOrScorePhrase( const TargetPhrase& targetP return score; } -void GlobalLexicalModel::Evaluate(const TargetPhrase& targetPhrase, ScoreComponentCollection* accumulator) const + void GlobalLexicalModel::Evaluate + (const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const { - accumulator->PlusEquals( this, GetFromCacheOrScorePhrase( targetPhrase ) ); + accumulator->PlusEquals( this, + GetFromCacheOrScorePhrase(context.GetTargetPhrase()) ); } - } diff --git a/moses/src/GlobalLexicalModel.h b/moses/src/GlobalLexicalModel.h index 88db637e8..7ab9459e5 100644 --- a/moses/src/GlobalLexicalModel.h +++ b/moses/src/GlobalLexicalModel.h @@ -50,7 +50,6 @@ private: #else std::auto_ptr<ThreadLocalStorage> m_local; #endif - Word *m_bias; FactorMask m_inputFactors; @@ -64,19 +63,10 @@ private: float GetFromCacheOrScorePhrase( const TargetPhrase& targetPhrase ) const; public: - GlobalLexicalModel(const std::string &filePath, - const float weight, - const std::vector< FactorType >& inFactors, - const std::vector< FactorType >& outFactors); - virtual ~GlobalLexicalModel(); - - virtual size_t GetNumScoreComponents() const { - return 1; - }; - - virtual std::string GetScoreProducerDescription(unsigned) const { - return "GlobalLexicalModel"; - }; + GlobalLexicalModel(const std::string &filePath, + const std::vector< FactorType >& inFactors, + const std::vector< FactorType >& outFactors); + virtual ~GlobalLexicalModel(); virtual std::string GetScoreProducerWeightShortName(unsigned) const { return "lex"; @@ -84,7 +74,17 @@ public: void InitializeForInput( Sentence const& in ); - void Evaluate(const TargetPhrase&, ScoreComponentCollection* ) const; + void Evaluate(const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const; + + + void EvaluateChart( + const ChartBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const + { + std::cerr << "EvaluateChart not implemented." << std::endl; + exit(1); + } }; } diff --git a/moses/src/GlobalLexicalModelUnlimited.cpp b/moses/src/GlobalLexicalModelUnlimited.cpp new file mode 100644 index 000000000..f1de65bd0 --- /dev/null +++ b/moses/src/GlobalLexicalModelUnlimited.cpp @@ -0,0 +1,280 @@ +#include "GlobalLexicalModelUnlimited.h" +#include <fstream> +#include "StaticData.h" +#include "InputFileStream.h" +#include "UserMessage.h" + +using namespace std; + +namespace Moses +{ + +bool GlobalLexicalModelUnlimited::Load(const std::string &filePathSource, + const std::string &filePathTarget) +{ + // restricted source word vocabulary + ifstream inFileSource(filePathSource.c_str()); + if (!inFileSource) + { + cerr << "could not open file " << filePathSource << endl; + return false; + } + + std::string line; + while (getline(inFileSource, line)) { + m_vocabSource.insert(line); + } + + inFileSource.close(); + + // restricted target word vocabulary + ifstream inFileTarget(filePathTarget.c_str()); + if (!inFileTarget) + { + cerr << "could not open file " << filePathTarget << endl; + return false; + } + + while (getline(inFileTarget, line)) { + m_vocabTarget.insert(line); + } + + inFileTarget.close(); + + m_unrestricted = false; + return true; +} + +void GlobalLexicalModelUnlimited::InitializeForInput( Sentence const& in ) +{ + m_local.reset(new ThreadLocalStorage); + m_local->input = ∈ +} + +void GlobalLexicalModelUnlimited::Evaluate(const Hypothesis& cur_hypo, ScoreComponentCollection* accumulator) const +{ + const Sentence& input = *(m_local->input); + const TargetPhrase& targetPhrase = cur_hypo.GetCurrTargetPhrase(); + + for(int targetIndex = 0; targetIndex < targetPhrase.GetSize(); targetIndex++ ) { + string targetString = targetPhrase.GetWord(targetIndex).GetString(0); // TODO: change for other factors + + if (m_ignorePunctuation) { + // check if first char is punctuation + char firstChar = targetString.at(0); + CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar ); + if(charIterator != m_punctuationHash.end()) + continue; + } + + if (m_biasFeature) { + stringstream feature; + feature << "glm_"; + feature << targetString; + feature << "~"; + feature << "**BIAS**"; + accumulator->SparsePlusEquals(feature.str(), 1); + } + + StringHash alreadyScored; + for(int sourceIndex = 0; sourceIndex < input.GetSize(); sourceIndex++ ) { + string sourceString = input.GetWord(sourceIndex).GetString(0); // TODO: change for other factors + + if (m_ignorePunctuation) { + // check if first char is punctuation + char firstChar = sourceString.at(0); + CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar ); + if(charIterator != m_punctuationHash.end()) + continue; + } + + if ( alreadyScored.find(sourceString) == alreadyScored.end()) { + bool sourceExists, targetExists; + if (!m_unrestricted) { + sourceExists = m_vocabSource.find( sourceString ) != m_vocabSource.end(); + targetExists = m_vocabTarget.find( targetString) != m_vocabTarget.end(); + } + + // no feature if vocab is in use and both words are not in restricted vocabularies + if (m_unrestricted || (sourceExists && targetExists)) { + if (m_sourceContext) { + if (sourceIndex == 0) { + // add <s> trigger feature for source + stringstream feature; + feature << "glm_"; + feature << targetString; + feature << "~"; + feature << "<s>,"; + feature << sourceString; + accumulator->SparsePlusEquals(feature.str(), 1); + alreadyScored[sourceString] = 1; + } + + // add source words to the right of current source word as context + for(int contextIndex = sourceIndex+1; contextIndex < input.GetSize(); contextIndex++ ) { + string contextString = input.GetWord(contextIndex).GetString(0); // TODO: change for other factors + bool contextExists; + if (!m_unrestricted) + contextExists = m_vocabSource.find( contextString ) != m_vocabSource.end(); + + if (m_unrestricted || contextExists) { + stringstream feature; + feature << "glm_"; + feature << targetString; + feature << "~"; + feature << sourceString; + feature << ","; + feature << contextString; + accumulator->SparsePlusEquals(feature.str(), 1); + alreadyScored[sourceString] = 1; + } + } + } + else if (m_biphrase) { + // --> look backwards for constructing context + int globalTargetIndex = cur_hypo.GetSize() - targetPhrase.GetSize() + targetIndex; + + // 1) source-target pair, trigger source word (can be discont.) and adjacent target word (bigram) + string targetContext; + if (globalTargetIndex > 0) + targetContext = cur_hypo.GetWord(globalTargetIndex-1).GetString(0); // TODO: change for other factors + else + targetContext = "<s>"; + + if (sourceIndex == 0) { + string sourceTrigger = "<s>"; + AddFeature(accumulator, alreadyScored, sourceTrigger, sourceString, + targetContext, targetString); + } + else + for(int contextIndex = sourceIndex-1; contextIndex >= 0; contextIndex-- ) { + string sourceTrigger = input.GetWord(contextIndex).GetString(0); // TODO: change for other factors + bool sourceTriggerExists = false; + if (!m_unrestricted) + sourceTriggerExists = m_vocabSource.find( sourceTrigger ) != m_vocabSource.end(); + + if (m_unrestricted || sourceTriggerExists) + AddFeature(accumulator, alreadyScored, sourceTrigger, sourceString, + targetContext, targetString); + } + + // 2) source-target pair, adjacent source word (bigram) and trigger target word (can be discont.) + string sourceContext; + if (sourceIndex-1 >= 0) + sourceContext = input.GetWord(sourceIndex-1).GetString(0); // TODO: change for other factors + else + sourceContext = "<s>"; + + if (globalTargetIndex == 0) { + string targetTrigger = "<s>"; + AddFeature(accumulator, alreadyScored, sourceContext, sourceString, + targetTrigger, targetString); + } + else + for(int globalContextIndex = globalTargetIndex-1; globalContextIndex >= 0; globalContextIndex-- ) { + string targetTrigger = cur_hypo.GetWord(globalContextIndex).GetString(0); // TODO: change for other factors + bool targetTriggerExists = false; + if (!m_unrestricted) + targetTriggerExists = m_vocabTarget.find( targetTrigger ) != m_vocabTarget.end(); + + if (m_unrestricted || targetTriggerExists) + AddFeature(accumulator, alreadyScored, sourceContext, sourceString, + targetTrigger, targetString); + } + } + else if (m_bitrigger) { + // allow additional discont. triggers on both sides + int globalTargetIndex = cur_hypo.GetSize() - targetPhrase.GetSize() + targetIndex; + + if (sourceIndex == 0) { + string sourceTrigger = "<s>"; + bool sourceTriggerExists = true; + + if (globalTargetIndex == 0) { + string targetTrigger = "<s>"; + bool targetTriggerExists = true; + + if (m_unrestricted || (sourceTriggerExists && targetTriggerExists)) + AddFeature(accumulator, alreadyScored, sourceTrigger, sourceString, + targetTrigger, targetString); + } + else { + // iterate backwards over target + for(int globalContextIndex = globalTargetIndex-1; globalContextIndex >= 0; globalContextIndex-- ) { + string targetTrigger = cur_hypo.GetWord(globalContextIndex).GetString(0); // TODO: change for other factors + bool targetTriggerExists = false; + if (!m_unrestricted) + targetTriggerExists = m_vocabTarget.find( targetTrigger ) != m_vocabTarget.end(); + + if (m_unrestricted || (sourceTriggerExists && targetTriggerExists)) + AddFeature(accumulator, alreadyScored, sourceTrigger, sourceString, + targetTrigger, targetString); + } + } + } + // iterate over both source and target + else { + // iterate backwards over source + for(int contextIndex = sourceIndex-1; contextIndex >= 0; contextIndex-- ) { + string sourceTrigger = input.GetWord(contextIndex).GetString(0); // TODO: change for other factors + bool sourceTriggerExists = false; + if (!m_unrestricted) + sourceTriggerExists = m_vocabSource.find( sourceTrigger ) != m_vocabSource.end(); + + if (globalTargetIndex == 0) { + string targetTrigger = "<s>"; + bool targetTriggerExists = true; + + if (m_unrestricted || (sourceTriggerExists && targetTriggerExists)) + AddFeature(accumulator, alreadyScored, sourceTrigger, sourceString, + targetTrigger, targetString); + } + else { + // iterate backwards over target + for(int globalContextIndex = globalTargetIndex-1; globalContextIndex >= 0; globalContextIndex-- ) { + string targetTrigger = cur_hypo.GetWord(globalContextIndex).GetString(0); // TODO: change for other factors + bool targetTriggerExists = false; + if (!m_unrestricted) + targetTriggerExists = m_vocabTarget.find( targetTrigger ) != m_vocabTarget.end(); + + if (m_unrestricted || (sourceTriggerExists && targetTriggerExists)) + AddFeature(accumulator, alreadyScored, sourceTrigger, sourceString, + targetTrigger, targetString); + } + } + } + } + } + else { + stringstream feature; + feature << "glm_"; + feature << targetString; + feature << "~"; + feature << sourceString; + accumulator->SparsePlusEquals(feature.str(), 1); + //alreadyScored.insert( &inputWord ); + alreadyScored[sourceString] = 1; + } + } + } + } + } +} + +void GlobalLexicalModelUnlimited::AddFeature(ScoreComponentCollection* accumulator, + StringHash alreadyScored, string sourceTrigger, string sourceWord, string targetTrigger, + string targetWord) const { + stringstream feature; + feature << "glm_"; + feature << targetTrigger; + feature << ","; + feature << targetWord; + feature << "~"; + feature << sourceTrigger; + feature << ","; + feature << sourceWord; + accumulator->SparsePlusEquals(feature.str(), 1); + alreadyScored[sourceWord] = 1; +} + +} diff --git a/moses/src/GlobalLexicalModelUnlimited.h b/moses/src/GlobalLexicalModelUnlimited.h new file mode 100644 index 000000000..307461db0 --- /dev/null +++ b/moses/src/GlobalLexicalModelUnlimited.h @@ -0,0 +1,146 @@ +#ifndef GLOBALLEXICALMODELUNLIMITED_H_ +#define GLOBALLEXICALMODELUNLIMITED_H_ + +#include <string> +#include <vector> +#include "Factor.h" +#include "Phrase.h" +#include "TypeDef.h" +#include "Util.h" +#include "WordsRange.h" +#include "ScoreProducer.h" +#include "FeatureFunction.h" +#include "FactorTypeSet.h" +#include "Sentence.h" + +#include "FFState.h" + +#ifdef WITH_THREADS +#include <boost/thread/tss.hpp> +#endif + +namespace Moses +{ + +class Factor; +class Phrase; +class Hypothesis; +class InputType; + +/** Discriminatively trained global lexicon model + * This is a implementation of Mauser et al., 2009's model that predicts + * each output word from _all_ the input words. The intuition behind this + * feature is that it uses context words for disambiguation + */ + +class GlobalLexicalModelUnlimited : public StatelessFeatureFunction +{ + typedef std::map< char, short > CharHash; + typedef std::map< std::string, short > StringHash; + + struct ThreadLocalStorage + { + const Sentence *input; + }; + +private: +#ifdef WITH_THREADS + boost::thread_specific_ptr<ThreadLocalStorage> m_local; +#else + std::auto_ptr<ThreadLocalStorage> m_local; +#endif + + CharHash m_punctuationHash; + + std::vector< FactorType > m_inputFactors; + std::vector< FactorType > m_outputFactors; + bool m_unrestricted; + + bool m_sourceContext; + bool m_biphrase; + bool m_bitrigger; + + bool m_biasFeature; + float m_sparseProducerWeight; + bool m_ignorePunctuation; + + std::set<std::string> m_vocabSource; + std::set<std::string> m_vocabTarget; + +public: + GlobalLexicalModelUnlimited(const std::vector< FactorType >& inFactors, const std::vector< FactorType >& outFactors, + bool biasFeature, bool ignorePunctuation, size_t context): + StatelessFeatureFunction("glm",ScoreProducer::unlimited), + m_inputFactors(inFactors), + m_outputFactors(outFactors), + m_unrestricted(true), + m_sourceContext(false), + m_biphrase(false), + m_bitrigger(false), + m_biasFeature(biasFeature), + m_sparseProducerWeight(1), + m_ignorePunctuation(ignorePunctuation) + { + std::cerr << "Creating global lexical model unlimited.. "; + + + switch(context) { + case 1: + m_sourceContext = true; + std::cerr << "using source context.. "; + break; + case 2: + m_biphrase = true; + std::cerr << "using biphrases.. "; + break; + case 3: + std::cerr << "using bitriggers.. "; + m_bitrigger = true; + break; + } + + // compile a list of punctuation characters + if (m_ignorePunctuation) { + std::cerr << "ignoring punctuation.. "; + char punctuation[] = "\"'!?¿·()#_,.:;•&@‑/\\0123456789~="; + for (size_t i=0; i < sizeof(punctuation)-1; ++i) + m_punctuationHash[punctuation[i]] = 1; + } + std::cerr << "done." << std::endl; + } + + std::string GetScoreProducerWeightShortName(unsigned) const { + return "glm"; + }; + + bool Load(const std::string &filePathSource, const std::string &filePathTarget); + + void InitializeForInput( Sentence const& in ); + + const FFState* EmptyHypothesisState(const InputType &) const { + return new DummyState(); + } + + //TODO: This implements the old interface, but cannot be updated because + //it appears to be stateful + void Evaluate(const Hypothesis& cur_hypo, + ScoreComponentCollection* accumulator) const; + + void EvaluateChart(const ChartHypothesis& /* cur_hypo */, + int /* featureID */, + ScoreComponentCollection* ) const { + /* Not implemented */ + assert(0); + } + + + void SetSparseProducerWeight(float weight) { m_sparseProducerWeight = weight; } + float GetSparseProducerWeight() const { return m_sparseProducerWeight; } + + void AddFeature(ScoreComponentCollection* accumulator, StringHash alreadyScored, + std::string sourceTrigger, std::string sourceWord, std::string targetTrigger, + std::string targetWord) const; +}; + +} +#endif /* GLOBALLEXICALMODELUNLIMITED_H_ */ diff --git a/moses/src/Hypothesis.cpp b/moses/src/Hypothesis.cpp index 530665f60..90083fc08 100644 --- a/moses/src/Hypothesis.cpp +++ b/moses/src/Hypothesis.cpp @@ -91,7 +91,6 @@ Hypothesis::Hypothesis(const Hypothesis &prevHypo, const TranslationOption &tran , m_wordDeleted(false) , m_totalScore(0.0f) , m_futureScore(0.0f) - , m_scoreBreakdown (prevHypo.m_scoreBreakdown) , m_ffStates(prevHypo.m_ffStates.size()) , m_arcList(NULL) , m_transOpt(&transOpt) @@ -247,18 +246,19 @@ int Hypothesis::RecombineCompare(const Hypothesis &compare) const } if (comp != 0) return comp; } - + return 0; } void Hypothesis::ResetScore() { - m_scoreBreakdown.ZeroAll(); + m_currScoreBreakdown.ZeroAll(); + m_scoreBreakdown.reset(0); m_futureScore = m_totalScore = 0.0f; } void Hypothesis::IncorporateTransOptScores() { - m_scoreBreakdown.PlusEquals(m_transOpt->GetScoreBreakdown()); + m_currScoreBreakdown.PlusEquals(m_transOpt->GetScoreBreakdown()); } void Hypothesis::EvaluateWith(StatefulFeatureFunction* sfff, @@ -266,12 +266,12 @@ void Hypothesis::EvaluateWith(StatefulFeatureFunction* sfff, m_ffStates[state_idx] = sfff->Evaluate( *this, m_prevHypo ? m_prevHypo->m_ffStates[state_idx] : NULL, - &m_scoreBreakdown); + &m_currScoreBreakdown); } void Hypothesis::EvaluateWith(const StatelessFeatureFunction* slff) { - slff->Evaluate(m_targetPhrase, &m_scoreBreakdown); + slff->Evaluate(PhraseBasedFeatureContext(this), &m_currScoreBreakdown); } void Hypothesis::CalculateFutureScore(const SquareMatrix& futureScore) { @@ -279,7 +279,7 @@ void Hypothesis::CalculateFutureScore(const SquareMatrix& futureScore) { } void Hypothesis::CalculateFinalScore() { - m_totalScore = m_scoreBreakdown.InnerProduct( + m_totalScore = GetScoreBreakdown().InnerProduct( StaticData::Instance().GetAllWeights()) + m_futureScore; } @@ -292,17 +292,24 @@ void Hypothesis::CalcScore(const SquareMatrix &futureScore) // option: add these here // language model scores for n-grams completely contained within a target // phrase are also included here - m_scoreBreakdown.PlusEquals(m_transOpt->GetScoreBreakdown()); + m_currScoreBreakdown = m_transOpt->GetScoreBreakdown(); + + // other stateless features have their scores cached in the + // TranslationOptionsCollection + m_manager.getSntTranslationOptions()->InsertPreCalculatedScores + (*m_transOpt, &m_currScoreBreakdown); const StaticData &staticData = StaticData::Instance(); clock_t t=0; // used to track time // compute values of stateless feature functions that were not - // cached in the translation option-- there is no principled distinction + // cached in the translation option const vector<const StatelessFeatureFunction*>& sfs = m_manager.GetTranslationSystem()->GetStatelessFeatureFunctions(); for (unsigned i = 0; i < sfs.size(); ++i) { - sfs[i]->Evaluate(m_targetPhrase, &m_scoreBreakdown); + if (!sfs[i]->ComputeValueInTranslationOption()) { + EvaluateWith(sfs[i]); + } } const vector<const StatefulFeatureFunction*>& ffs = @@ -311,7 +318,7 @@ void Hypothesis::CalcScore(const SquareMatrix &futureScore) m_ffStates[i] = ffs[i]->Evaluate( *this, m_prevHypo ? m_prevHypo->m_ffStates[i] : NULL, - &m_scoreBreakdown); + &m_currScoreBreakdown); } IFVERBOSE(2) { @@ -321,8 +328,19 @@ void Hypothesis::CalcScore(const SquareMatrix &futureScore) // FUTURE COST m_futureScore = futureScore.CalcFutureScore( m_sourceCompleted ); + // Apply sparse producer weights + ScoreComponentCollection tempScoreBreakdown = m_currScoreBreakdown; + const vector<const FeatureFunction*>& sparseProducers = m_manager.GetTranslationSystem()->GetSparseProducers(); + for (unsigned i = 0; i < sparseProducers.size(); ++i) { + float weight = sparseProducers[i]->GetSparseProducerWeight(); + tempScoreBreakdown.MultiplyEquals(sparseProducers[i], weight); + } + // TOTAL - m_totalScore = m_scoreBreakdown.InnerProduct(staticData.GetAllWeights()) + m_futureScore; + m_totalScore = tempScoreBreakdown.InnerProduct(staticData.GetAllWeights()) + m_futureScore; + if (m_prevHypo) { + m_totalScore += m_prevHypo->m_totalScore - m_prevHypo->m_futureScore; + } IFVERBOSE(2) { m_manager.GetSentenceStats().AddTimeOtherScore( clock()-t ); @@ -353,7 +371,7 @@ float Hypothesis::CalcExpectedScore( const SquareMatrix &futureScore ) m_futureScore = futureScore.CalcFutureScore( m_sourceCompleted ); // TOTAL - float total = m_scoreBreakdown.InnerProduct(staticData.GetAllWeights()) + m_futureScore + estimatedLMScore; + float total = m_totalScore + estimatedLMScore; IFVERBOSE(2) { m_manager.GetSentenceStats().AddTimeEstimateScore( clock()-t ); @@ -375,11 +393,14 @@ void Hypothesis::CalcRemainingScore() } // WORD PENALTY - m_scoreBreakdown.PlusEquals(m_manager.GetTranslationSystem()->GetWordPenaltyProducer() + m_currScoreBreakdown.PlusEquals(m_manager.GetTranslationSystem()->GetWordPenaltyProducer() , - (float)m_currTargetWordsRange.GetNumWordsCovered()); // TOTAL - m_totalScore = m_scoreBreakdown.InnerProduct(staticData.GetAllWeights()) + m_futureScore; + m_totalScore = m_currScoreBreakdown.InnerProduct(staticData.GetAllWeights()) + m_futureScore; + if (m_prevHypo) { + m_totalScore += m_prevHypo->m_totalScore - m_prevHypo->m_futureScore; + } IFVERBOSE(2) { m_manager.GetSentenceStats().AddTimeOtherScore( clock()-t ); @@ -424,7 +445,7 @@ void Hypothesis::PrintHypothesis() const // TRACE_ERR( "\tlanguage model cost "); // <<m_score[ScoreType::LanguageModelScore]<<endl; // TRACE_ERR( "\tword penalty "); // <<(m_score[ScoreType::WordPenalty]*weightWordPenalty)<<endl; TRACE_ERR( "\tscore "<<m_totalScore - m_futureScore<<" + future cost "<<m_futureScore<<" = "<<m_totalScore<<endl); - TRACE_ERR( "\tunweighted feature scores: " << m_scoreBreakdown << endl); + TRACE_ERR( "\tunweighted feature scores: " << m_currScoreBreakdown << endl); //PrintLMScores(); } diff --git a/moses/src/Hypothesis.h b/moses/src/Hypothesis.h index da722b751..fe885ccc8 100644 --- a/moses/src/Hypothesis.h +++ b/moses/src/Hypothesis.h @@ -24,6 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #define moses_Hypothesis_h #include <iostream> +#include <memory> #include <vector> #include "Phrase.h" #include "TypeDef.h" @@ -78,7 +79,8 @@ protected: bool m_wordDeleted; float m_totalScore; /*! score so far */ float m_futureScore; /*! estimated future cost to translate rest of sentence */ - ScoreComponentCollection m_scoreBreakdown; /*! detailed score break-down by components (for instance language model, word penalty, etc) */ + mutable std::auto_ptr<ScoreComponentCollection> m_scoreBreakdown; /*! detailed score break-down by components (for instance language model, word penalty, etc) */ + ScoreComponentCollection m_currScoreBreakdown; /*! scores for this hypothesis */ std::vector<const FFState*> m_ffStates; const Hypothesis *m_winningHypo; ArcList *m_arcList; /*! all arcs that end at the same trellis point as this hypothesis */ @@ -211,6 +213,19 @@ public: out << (Phrase) GetCurrTargetPhrase(); } + void ToStringStream(std::stringstream& out) const { + if (m_prevHypo != NULL) { + m_prevHypo->ToStream(out); + } + out << (Phrase) GetCurrTargetPhrase(); + } + + std::string GetOutputString() const { + std::stringstream out; + ToStringStream(out); + return out.str(); + } + TO_STRING(); inline void SetWinningHypo(const Hypothesis *hypo) { @@ -228,7 +243,13 @@ public: return m_arcList; } const ScoreComponentCollection& GetScoreBreakdown() const { - return m_scoreBreakdown; + if (!m_scoreBreakdown.get()) { + m_scoreBreakdown.reset(new ScoreComponentCollection(m_currScoreBreakdown)); + if (m_prevHypo) { + m_scoreBreakdown->PlusEquals(m_prevHypo->GetScoreBreakdown()); + } + } + return *m_scoreBreakdown; } float GetTotalScore() const { return m_totalScore; diff --git a/moses/src/InputType.h b/moses/src/InputType.h index e3b2f108e..b72cccfd8 100644 --- a/moses/src/InputType.h +++ b/moses/src/InputType.h @@ -47,9 +47,15 @@ class InputType { protected: long m_translationId; //< contiguous Id + long m_documentId; + long m_topicId; + std::vector<std::string> m_topicIdAndProb; + bool m_useTopicId; + bool m_useTopicIdAndProb; bool m_hasMetaData; long m_segId; ReorderingConstraint m_reorderingConstraint; /**< limits on reordering specified either by "-mp" switch or xml tags */ + std::string m_textType; public: @@ -70,6 +76,42 @@ public: void SetTranslationId(long translationId) { m_translationId = translationId; } + long GetDocumentId() const { + return m_documentId; + } + void SetDocumentId(long documentId) { + m_documentId = documentId; + } + long GetTopicId() const { + return m_topicId; + } + void SetTopicId(long topicId) { + m_topicId = topicId; + } + const std::vector<std::string>* GetTopicIdAndProb() const { + return &m_topicIdAndProb; + } + void SetTopicIdAndProb(std::vector<std::string> topicIdAndProb) { + m_topicIdAndProb = topicIdAndProb; + } + bool GetUseTopicId() const { + return m_useTopicId; + } + void SetUseTopicId(bool useTopicId) { + m_useTopicId = useTopicId; + } + bool GetUseTopicIdAndProb() const { + return m_useTopicIdAndProb; + } + void SetUseTopicIdAndProb(bool useTopicIdAndProb) { + m_useTopicIdAndProb = useTopicIdAndProb; + } + std::string GetTextType() const { + return m_textType; + } + void SetTextType(std::string type) { + m_textType = type; + } //! returns the number of words moved virtual int ComputeDistortionDistance(const WordsRange& prev, const WordsRange& current) const; diff --git a/moses/src/Jamfile b/moses/src/Jamfile index 1ea22eb81..d88af7ba9 100644 --- a/moses/src/Jamfile +++ b/moses/src/Jamfile @@ -21,9 +21,13 @@ if $(have-clock[2]) = 0 { lib moses_internal : #All cpp files except those listed -[ glob *.cpp DynSAInclude/*.cpp : PhraseDictionary.cpp ThreadPool.cpp SyntacticLanguageModel.cpp ] +[ glob *.cpp DynSAInclude/*.cpp : PhraseDictionary.cpp ThreadPool.cpp SyntacticLanguageModel.cpp *Test.cpp Mock*.cpp ] synlm ThreadPool headers rt ; lib moses : PhraseDictionary.cpp moses_internal CYKPlusParser//CYKPlusParser CompactPT//CompactPT LM//LM RuleTable//RuleTable Scope3Parser//Scope3Parser fuzzy-match//fuzzy-match headers ../..//z ../../OnDiskPt//OnDiskPt ; alias headers-to-install : [ glob-tree *.h ] ; + +import testing ; + +unit-test moses_test : [ glob *Test.cpp Mock*.cpp ] moses ../..//boost_unit_test_framework ; diff --git a/moses/src/LM/Base.cpp b/moses/src/LM/Base.cpp index e3f011663..45edf973e 100644 --- a/moses/src/LM/Base.cpp +++ b/moses/src/LM/Base.cpp @@ -32,36 +32,26 @@ using namespace std; namespace Moses { -LanguageModel::LanguageModel() { +LanguageModel::LanguageModel() : + StatefulFeatureFunction("LM", StaticData::Instance().GetLMEnableOOVFeature() ? 2 : 1 ) { m_enableOOVFeature = StaticData::Instance().GetLMEnableOOVFeature(); } -void LanguageModel::Init(ScoreIndexManager &scoreIndexManager) { - scoreIndexManager.AddScoreProducer(this); -} LanguageModel::~LanguageModel() {} -// don't inline virtual funcs... -size_t LanguageModel::GetNumScoreComponents() const { - if (m_enableOOVFeature) { - return 2; - } else { - return 1; - } -} - float LanguageModel::GetWeight() const { - size_t lmIndex = StaticData::Instance().GetScoreIndexManager(). - GetBeginIndex(GetScoreBookkeepingID()); - return StaticData::Instance().GetAllWeights()[lmIndex]; + //return StaticData::Instance().GetAllWeights().GetScoresForProducer(this)[0]; + return StaticData::Instance().GetWeights(this)[0]; } float LanguageModel::GetOOVWeight() const { - if (!m_enableOOVFeature) return 0; - size_t lmIndex = StaticData::Instance().GetScoreIndexManager(). - GetBeginIndex(GetScoreBookkeepingID()); - return StaticData::Instance().GetAllWeights()[lmIndex+1]; + if (m_enableOOVFeature) { + //return StaticData::Instance().GetAllWeights().GetScoresForProducer(this)[1]; + return StaticData::Instance().GetWeights(this)[1]; + } else { + return 0; + } } } // namespace Moses diff --git a/moses/src/LM/Base.h b/moses/src/LM/Base.h index 51b651b8c..424dcb443 100644 --- a/moses/src/LM/Base.h +++ b/moses/src/LM/Base.h @@ -32,7 +32,6 @@ namespace Moses class FactorCollection; class Factor; class Phrase; -class ScoreIndexManager; //! Abstract base class which represent a language model on a contiguous phrase class LanguageModel : public StatefulFeatureFunction { @@ -40,7 +39,6 @@ protected: LanguageModel(); // This can't be in the constructor for virual function dispatch reasons - void Init(ScoreIndexManager &scoreIndexManager); bool m_enableOOVFeature; @@ -48,10 +46,7 @@ public: virtual ~LanguageModel(); // Make another feature without copying the underlying model data. - virtual LanguageModel *Duplicate(ScoreIndexManager &scoreIndexManager) const = 0; - - //! see ScoreProducer.h - std::size_t GetNumScoreComponents() const; + virtual LanguageModel *Duplicate() const = 0; bool OOVFeatureEnabled() const { return m_enableOOVFeature; diff --git a/moses/src/LM/Factory.cpp b/moses/src/LM/Factory.cpp index 3498be55e..0a3dd60b1 100644 --- a/moses/src/LM/Factory.cpp +++ b/moses/src/LM/Factory.cpp @@ -65,11 +65,10 @@ LanguageModel* CreateLanguageModel(LMImplementation lmImplementation , const std::vector<FactorType> &factorTypes , size_t nGramOrder , const std::string &languageModelFile - , ScoreIndexManager &scoreIndexManager - , int dub ) + , int dub) { if (lmImplementation == Ken || lmImplementation == LazyKen) { - return ConstructKenLM(languageModelFile, scoreIndexManager, factorTypes[0], lmImplementation == LazyKen); + return ConstructKenLM(languageModelFile, factorTypes[0], lmImplementation == LazyKen); } LanguageModelImplementation *lm = NULL; switch (lmImplementation) { @@ -140,7 +139,7 @@ LanguageModel* CreateLanguageModel(LMImplementation lmImplementation } } - return new LMRefCount(scoreIndexManager, lm); + return new LMRefCount(lm); } } diff --git a/moses/src/LM/Factory.h b/moses/src/LM/Factory.h index 89457887c..1a140846a 100644 --- a/moses/src/LM/Factory.h +++ b/moses/src/LM/Factory.h @@ -11,22 +11,19 @@ namespace Moses { class LanguageModel; -class ScoreIndexManager; - -namespace LanguageModelFactory -{ - -/** - * creates a language model that will use the appropriate - * language model toolkit as its underlying implementation - */ -LanguageModel* CreateLanguageModel(LMImplementation lmImplementation - , const std::vector<FactorType> &factorTypes - , size_t nGramOrder - , const std::string &languageModelFile - , ScoreIndexManager &scoreIndexManager - , int dub); +namespace LanguageModelFactory { + + /** + * creates a language model that will use the appropriate + * language model toolkit as its underlying implementation + */ + LanguageModel* CreateLanguageModel(LMImplementation lmImplementation + , const std::vector<FactorType> &factorTypes + , size_t nGramOrder + , const std::string &languageModelFile + , int dub); + }; } diff --git a/moses/src/LM/Implementation.cpp b/moses/src/LM/Implementation.cpp index 6fb1ffbe0..94663cc9f 100644 --- a/moses/src/LM/Implementation.cpp +++ b/moses/src/LM/Implementation.cpp @@ -231,8 +231,8 @@ private: { const TargetPhrase &target = hypo.GetCurrTargetPhrase(); const AlignmentInfo::NonTermIndexMap &nonTermIndexMap = - target.GetAlignmentInfo().GetNonTermIndexMap(); - + target.GetAlignmentInfo().GetNonTermIndexMap(); + // loop over the rule that is being applied for (size_t pos = 0; pos < target.GetSize(); ++pos) { const Word &word = target.GetWord(pos); @@ -283,10 +283,11 @@ private: } // construct suffix analogous to prefix else { + const TargetPhrase& target = hypo.GetCurrTargetPhrase(); const AlignmentInfo::NonTermIndexMap &nonTermIndexMap = - hypo.GetCurrTargetPhrase().GetAlignmentInfo().GetNonTermIndexMap(); - for (int pos = (int) hypo.GetCurrTargetPhrase().GetSize() - 1; pos >= 0 ; --pos) { - const Word &word = hypo.GetCurrTargetPhrase().GetWord(pos); + target.GetAlignmentInfo().GetNonTermIndexMap(); + for (int pos = (int) target.GetSize() - 1; pos >= 0 ; --pos) { + const Word &word = target.GetWord(pos); if (word.IsNonTerminal()) { size_t nonTermInd = nonTermIndexMap[pos]; @@ -388,8 +389,9 @@ FFState* LanguageModelImplementation::EvaluateChart(const ChartHypothesis& hypo, float finalizedScore = 0.0; // finalized, has sufficient context // get index map for underlying hypotheses + const TargetPhrase &target = hypo.GetCurrTargetPhrase(); const AlignmentInfo::NonTermIndexMap &nonTermIndexMap = - hypo.GetCurrTargetPhrase().GetAlignmentInfo().GetNonTermIndexMap(); + hypo.GetCurrTargetPhrase().GetAlignmentInfo().GetNonTermIndexMap(); // loop over rule for (size_t phrasePos = 0, wordPos = 0; diff --git a/moses/src/LM/Implementation.h b/moses/src/LM/Implementation.h index b7771f403..fe1a406da 100644 --- a/moses/src/LM/Implementation.h +++ b/moses/src/LM/Implementation.h @@ -114,9 +114,6 @@ public: return m_sentenceEndArray; } - virtual std::string GetScoreProducerDescription(unsigned) const = 0; - - float GetWeight() const; std::string GetScoreProducerWeightShortName(unsigned) const { return "lm"; @@ -129,12 +126,10 @@ public: class LMRefCount : public LanguageModel { public: - LMRefCount(ScoreIndexManager &scoreIndexManager, LanguageModelImplementation *impl) : m_impl(impl) { - Init(scoreIndexManager); - } + LMRefCount(LanguageModelImplementation *impl) : m_impl(impl) {} - LanguageModel *Duplicate(ScoreIndexManager &scoreIndexManager) const { - return new LMRefCount(scoreIndexManager, *this); + LanguageModel *Duplicate() const { + return new LMRefCount(*this); } void InitializeBeforeSentenceProcessing() { @@ -165,16 +160,11 @@ class LMRefCount : public LanguageModel { return m_impl->EvaluateChart(cur_hypo, featureID, accumulator, this); } - std::string GetScoreProducerDescription(unsigned int param) const { - return m_impl->GetScoreProducerDescription(param); - } LanguageModelImplementation *MosesServerCppShouldNotHaveLMCode() { return m_impl.get(); } private: - LMRefCount(ScoreIndexManager &scoreIndexManager, const LMRefCount ©_from) : m_impl(copy_from.m_impl) { - Init(scoreIndexManager); - } + LMRefCount(const LMRefCount ©_from) : m_impl(copy_from.m_impl) {} boost::shared_ptr<LanguageModelImplementation> m_impl; }; diff --git a/moses/src/LM/Ken.cpp b/moses/src/LM/Ken.cpp index 28d0534cd..2d65ad7df 100644 --- a/moses/src/LM/Ken.cpp +++ b/moses/src/LM/Ken.cpp @@ -61,9 +61,9 @@ struct KenLMState : public FFState { */ template <class Model> class LanguageModelKen : public LanguageModel { public: - LanguageModelKen(const std::string &file, ScoreIndexManager &manager, FactorType factorType, bool lazy); + LanguageModelKen(const std::string &file, FactorType factorType, bool lazy); - LanguageModel *Duplicate(ScoreIndexManager &scoreIndexManager) const; + LanguageModel *Duplicate() const; bool Useable(const Phrase &phrase) const { return (phrase.GetSize()>0 && phrase.GetFactor(0, m_factorType) != NULL); @@ -88,7 +88,7 @@ template <class Model> class LanguageModelKen : public LanguageModel { FFState *EvaluateChart(const ChartHypothesis& cur_hypo, int featureID, ScoreComponentCollection *accumulator) const; private: - LanguageModelKen(ScoreIndexManager &manager, const LanguageModelKen<Model> ©_from); + LanguageModelKen(const LanguageModelKen<Model> ©_from); lm::WordIndex TranslateID(const Word &word) const { std::size_t factor = word.GetFactor(m_factorType)->GetId(); @@ -138,7 +138,7 @@ private: std::vector<lm::WordIndex> &m_mapping; }; -template <class Model> LanguageModelKen<Model>::LanguageModelKen(const std::string &file, ScoreIndexManager &manager, FactorType factorType, bool lazy) : m_factorType(factorType) { +template <class Model> LanguageModelKen<Model>::LanguageModelKen(const std::string &file, FactorType factorType, bool lazy) : m_factorType(factorType) { lm::ngram::Config config; IFVERBOSE(1) { config.messages = &std::cerr; @@ -153,20 +153,18 @@ template <class Model> LanguageModelKen<Model>::LanguageModelKen(const std::stri m_ngram.reset(new Model(file.c_str(), config)); m_beginSentenceFactor = collection.AddFactor(BOS_); - Init(manager); } -template <class Model> LanguageModel *LanguageModelKen<Model>::Duplicate(ScoreIndexManager &manager) const { - return new LanguageModelKen<Model>(manager, *this); +template <class Model> LanguageModel *LanguageModelKen<Model>::Duplicate() const { + return new LanguageModelKen<Model>(*this); } -template <class Model> LanguageModelKen<Model>::LanguageModelKen(ScoreIndexManager &manager, const LanguageModelKen<Model> ©_from) : +template <class Model> LanguageModelKen<Model>::LanguageModelKen(const LanguageModelKen<Model> ©_from) : m_ngram(copy_from.m_ngram), // TODO: don't copy this. m_lmIdLookup(copy_from.m_lmIdLookup), m_factorType(copy_from.m_factorType), m_beginSentenceFactor(copy_from.m_beginSentenceFactor) { - Init(manager); } template <class Model> void LanguageModelKen<Model>::CalcScore(const Phrase &phrase, float &fullScore, float &ngramScore, size_t &oovCount) const { @@ -286,7 +284,9 @@ class LanguageModelChartStateKenLM : public FFState { template <class Model> FFState *LanguageModelKen<Model>::EvaluateChart(const ChartHypothesis& hypo, int featureID, ScoreComponentCollection *accumulator) const { LanguageModelChartStateKenLM *newState = new LanguageModelChartStateKenLM(); lm::ngram::RuleScore<Model> ruleScore(*m_ngram, newState->GetChartState()); - const AlignmentInfo::NonTermIndexMap &nonTermIndexMap = hypo.GetCurrTargetPhrase().GetAlignmentInfo().GetNonTermIndexMap(); + const TargetPhrase &target = hypo.GetCurrTargetPhrase(); + const AlignmentInfo::NonTermIndexMap &nonTermIndexMap = + target.GetAlignmentInfo().GetNonTermIndexMap(); const size_t size = hypo.GetCurrTargetPhrase().GetSize(); size_t phrasePos = 0; @@ -323,29 +323,29 @@ template <class Model> FFState *LanguageModelKen<Model>::EvaluateChart(const Cha } // namespace -LanguageModel *ConstructKenLM(const std::string &file, ScoreIndexManager &manager, FactorType factorType, bool lazy) { +LanguageModel *ConstructKenLM(const std::string &file, FactorType factorType, bool lazy) { try { lm::ngram::ModelType model_type; if (lm::ngram::RecognizeBinary(file.c_str(), model_type)) { switch(model_type) { case lm::ngram::PROBING: - return new LanguageModelKen<lm::ngram::ProbingModel>(file, manager, factorType, lazy); + return new LanguageModelKen<lm::ngram::ProbingModel>(file, factorType, lazy); case lm::ngram::REST_PROBING: - return new LanguageModelKen<lm::ngram::RestProbingModel>(file, manager, factorType, lazy); + return new LanguageModelKen<lm::ngram::RestProbingModel>(file, factorType, lazy); case lm::ngram::TRIE: - return new LanguageModelKen<lm::ngram::TrieModel>(file, manager, factorType, lazy); + return new LanguageModelKen<lm::ngram::TrieModel>(file, factorType, lazy); case lm::ngram::QUANT_TRIE: - return new LanguageModelKen<lm::ngram::QuantTrieModel>(file, manager, factorType, lazy); + return new LanguageModelKen<lm::ngram::QuantTrieModel>(file, factorType, lazy); case lm::ngram::ARRAY_TRIE: - return new LanguageModelKen<lm::ngram::ArrayTrieModel>(file, manager, factorType, lazy); + return new LanguageModelKen<lm::ngram::ArrayTrieModel>(file, factorType, lazy); case lm::ngram::QUANT_ARRAY_TRIE: - return new LanguageModelKen<lm::ngram::QuantArrayTrieModel>(file, manager, factorType, lazy); + return new LanguageModelKen<lm::ngram::QuantArrayTrieModel>(file, factorType, lazy); default: std::cerr << "Unrecognized kenlm model type " << model_type << std::endl; abort(); } } else { - return new LanguageModelKen<lm::ngram::ProbingModel>(file, manager, factorType, lazy); + return new LanguageModelKen<lm::ngram::ProbingModel>(file, factorType, lazy); } } catch (std::exception &e) { std::cerr << e.what() << std::endl; diff --git a/moses/src/LM/Ken.h b/moses/src/LM/Ken.h index 1e02f5858..e2a8d613c 100644 --- a/moses/src/LM/Ken.h +++ b/moses/src/LM/Ken.h @@ -28,11 +28,10 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA namespace Moses { -class ScoreIndexManager; class LanguageModel; //! This will also load. Returns a templated KenLM class -LanguageModel *ConstructKenLM(const std::string &file, ScoreIndexManager &manager, FactorType factorType, bool lazy); +LanguageModel *ConstructKenLM(const std::string &file, FactorType factorType, bool lazy); } // namespace Moses diff --git a/moses/src/LM/MultiFactor.cpp b/moses/src/LM/MultiFactor.cpp index 672c924d2..d141b88b3 100644 --- a/moses/src/LM/MultiFactor.cpp +++ b/moses/src/LM/MultiFactor.cpp @@ -24,14 +24,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA namespace Moses { -std::string LanguageModelMultiFactor::GetScoreProducerDescription(unsigned) const -{ - std::ostringstream oss; - // what about LMs that are over multiple factors at once, POS + stem, for example? - oss << GetNGramOrder() << "-gram LM score, factor-type= ??? " << ", file=" << m_filePath; - return oss.str(); -} - bool LanguageModelMultiFactor::Useable(const Phrase &phrase) const { if (phrase.GetSize()==0) diff --git a/moses/src/LM/MultiFactor.h b/moses/src/LM/MultiFactor.h index 9bda8d3d9..f87d15db4 100644 --- a/moses/src/LM/MultiFactor.h +++ b/moses/src/LM/MultiFactor.h @@ -41,6 +41,8 @@ class LanguageModelMultiFactor : public LanguageModelImplementation protected: FactorMask m_factorTypes; + LanguageModelMultiFactor(){} + public: virtual bool Load(const std::string &filePath , const std::vector<FactorType> &factorTypes @@ -50,8 +52,7 @@ public: return MultiFactor; } - std::string GetScoreProducerDescription(unsigned) const; - bool Useable(const Phrase &phrase) const; + bool Useable(const Phrase &phrase) const; }; } diff --git a/moses/src/LM/SingleFactor.cpp b/moses/src/LM/SingleFactor.cpp index e7948c584..73c9e6e83 100644 --- a/moses/src/LM/SingleFactor.cpp +++ b/moses/src/LM/SingleFactor.cpp @@ -39,14 +39,6 @@ namespace Moses LanguageModelSingleFactor::~LanguageModelSingleFactor() {} -std::string LanguageModelSingleFactor::GetScoreProducerDescription(unsigned) const -{ - std::ostringstream oss; - // what about LMs that are over multiple factors at once, POS + stem, for example? - oss << "LM_" << GetNGramOrder() << "gram"; - return oss.str(); -} - struct PointerState : public FFState { const void* lmstate; PointerState(const void* lms) { diff --git a/moses/src/LM/SingleFactor.h b/moses/src/LM/SingleFactor.h index b5b79e2ff..44fdf9b16 100644 --- a/moses/src/LM/SingleFactor.h +++ b/moses/src/LM/SingleFactor.h @@ -38,30 +38,36 @@ protected: const Factor *m_sentenceStart, *m_sentenceEnd; FactorType m_factorType; + LanguageModelSingleFactor() {} + public: - virtual ~LanguageModelSingleFactor(); - virtual bool Load(const std::string &filePath - , FactorType factorType - , size_t nGramOrder) = 0; - - LMType GetLMType() const { - return SingleFactor; - } - - bool Useable(const Phrase &phrase) const { - return (phrase.GetSize()>0 && phrase.GetFactor(0, m_factorType) != NULL); - } - - const Factor *GetSentenceStart() const { - return m_sentenceStart; - } - const Factor *GetSentenceEnd() const { - return m_sentenceEnd; - } - FactorType GetFactorType() const { - return m_factorType; - } - std::string GetScoreProducerDescription(unsigned) const; + virtual ~LanguageModelSingleFactor(); + virtual bool Load(const std::string &filePath + , FactorType factorType + , size_t nGramOrder) = 0; + + LMType GetLMType() const + { + return SingleFactor; + } + + bool Useable(const Phrase &phrase) const + { + return (phrase.GetSize()>0 && phrase.GetFactor(0, m_factorType) != NULL); + } + + const Factor *GetSentenceStart() const + { + return m_sentenceStart; + } + const Factor *GetSentenceEnd() const + { + return m_sentenceEnd; + } + FactorType GetFactorType() const + { + return m_factorType; + } }; // Single factor LM that uses a null pointer state. diff --git a/moses/src/LMList.cpp b/moses/src/LMList.cpp index 8142b05b3..cac1199e0 100644 --- a/moses/src/LMList.cpp +++ b/moses/src/LMList.cpp @@ -73,14 +73,7 @@ void LMList::CalcScore(const Phrase &phrase, float &retFullScore, float &retNGra void LMList::Add(LanguageModel *lm) { - m_coll.push_back(lm); - - const ScoreIndexManager &scoreMgr = StaticData::Instance().GetScoreIndexManager(); - size_t startInd = scoreMgr.GetBeginIndex(lm->GetScoreBookkeepingID()) - ,endInd = scoreMgr.GetEndIndex(lm->GetScoreBookkeepingID()) - 1; - - m_minInd = min(m_minInd, startInd); - m_maxInd = max(m_maxInd, endInd); + m_coll.push_back(lm); } } diff --git a/moses/src/LMList.h b/moses/src/LMList.h index 8d14a4810..96ff7f7a7 100644 --- a/moses/src/LMList.h +++ b/moses/src/LMList.h @@ -15,15 +15,13 @@ class ScoreComponentCollection; class LMList { protected: - typedef std::list < LanguageModel* > CollType; - CollType m_coll; - - size_t m_minInd, m_maxInd; - + typedef std::list < LanguageModel* > CollType; + CollType m_coll; + public: - typedef CollType::iterator iterator; - typedef CollType::const_iterator const_iterator; - //! iterators + typedef CollType::iterator iterator; + typedef CollType::const_iterator const_iterator; + //! iterators const_iterator begin() const { return m_coll.begin(); } @@ -34,14 +32,12 @@ public: return m_coll.size(); } - LMList() - :m_minInd(std::numeric_limits<size_t>::max()) - ,m_maxInd(0) - {} - void CleanUp(); - ~LMList(); - - void CalcScore(const Phrase &phrase, float &retFullScore, float &retNGramScore, float &retOOVScore, ScoreComponentCollection* breakdown) const; + LMList() + {} + void CleanUp(); + ~LMList(); + + void CalcScore(const Phrase &phrase, float &retFullScore, float &retNGramScore, float &retOOVScore, ScoreComponentCollection* breakdown) const; void InitializeBeforeSentenceProcessing() { std::list<LanguageModel*>::iterator lm_iter; for (lm_iter = m_coll.begin(); @@ -59,14 +55,8 @@ public: } } - void Add(LanguageModel *lm); + void Add(LanguageModel *lm); - size_t GetMinIndex() const { - return m_minInd; - } - size_t GetMaxIndex() const { - return m_maxInd; - } }; diff --git a/moses/src/LexicalReordering.cpp b/moses/src/LexicalReordering.cpp index 0cafa8198..b32b96b0d 100644 --- a/moses/src/LexicalReordering.cpp +++ b/moses/src/LexicalReordering.cpp @@ -10,11 +10,14 @@ namespace Moses LexicalReordering::LexicalReordering(std::vector<FactorType>& f_factors, std::vector<FactorType>& e_factors, - const std::string &modelType, + const LexicalReorderingConfiguration& configuration, const std::string &filePath, const std::vector<float>& weights) - : m_configuration(this, modelType) + : StatefulFeatureFunction("LexicalReordering_" + configuration.GetModelString(), + configuration.GetNumScoreComponents()), + m_configuration(configuration) { + m_configuration.SetScoreProducer(this); std::cerr << "Creating lexical reordering...\n"; std::cerr << "weights: "; for(size_t w = 0; w < weights.size(); ++w) { @@ -22,7 +25,7 @@ LexicalReordering::LexicalReordering(std::vector<FactorType>& f_factors, } std::cerr << "\n"; - m_modelTypeString = modelType; + m_modelTypeString = m_configuration.GetModelString(); switch(m_configuration.GetCondition()) { case LexicalReorderingConfiguration::FE: @@ -46,17 +49,17 @@ LexicalReordering::LexicalReordering(std::vector<FactorType>& f_factors, exit(1); } - - if(weights.size() != m_configuration.GetNumScoreComponents()) { + size_t numberOfScoreComponents = m_configuration.GetNumScoreComponents(); + if (weights.size() > numberOfScoreComponents) { + m_configuration.SetAdditionalScoreComponents(weights.size() - numberOfScoreComponents); + } else if(weights.size() < numberOfScoreComponents) { std::ostringstream os; - os << "Lexical reordering model (type " << modelType << "): expected " << m_numScoreComponents << " weights, got " << weights.size() << std::endl; + os << "Lexical reordering model (type " << m_modelTypeString << "): expected " << numberOfScoreComponents << " weights, got " << weights.size() << std::endl; UserMessage::Add(os.str()); exit(1); } - // add ScoreProducer - don't do this before our object is set up - const_cast<ScoreIndexManager&>(StaticData::Instance().GetScoreIndexManager()).AddScoreProducer(this); - const_cast<StaticData&>(StaticData::Instance()).SetWeightsForScoreProducer(this, weights); + const_cast<StaticData&>(StaticData::Instance()).SetWeights(this, weights); m_table = LexicalReorderingTable::LoadAvailable(filePath, m_factorsF, m_factorsE, std::vector<FactorType>()); } diff --git a/moses/src/LexicalReordering.h b/moses/src/LexicalReordering.h index 13786a17a..f46679e4e 100644 --- a/moses/src/LexicalReordering.h +++ b/moses/src/LexicalReordering.h @@ -24,24 +24,32 @@ class InputType; /** implementation of lexical reordering (Tilman ...) for phrase-based decoding */ -class LexicalReordering : public StatefulFeatureFunction -{ -public: - LexicalReordering(std::vector<FactorType>& f_factors, - std::vector<FactorType>& e_factors, - const std::string &modelType, - const std::string &filePath, - const std::vector<float>& weights); - virtual ~LexicalReordering(); - - virtual size_t GetNumScoreComponents() const { - return m_configuration.GetNumScoreComponents(); - } - - virtual FFState* Evaluate(const Hypothesis& cur_hypo, - const FFState* prev_state, - ScoreComponentCollection* accumulator) const; - +class LexicalReordering : public StatefulFeatureFunction { +public: + LexicalReordering(std::vector<FactorType>& f_factors, + std::vector<FactorType>& e_factors, + const LexicalReorderingConfiguration& configuration, + const std::string &filePath, + const std::vector<float>& weights); + virtual ~LexicalReordering(); + + virtual FFState* Evaluate(const Hypothesis& cur_hypo, + const FFState* prev_state, + ScoreComponentCollection* accumulator) const; + + virtual const FFState* EmptyHypothesisState(const InputType &input) const; + + + std::string GetScoreProducerWeightShortName(unsigned) const { + return "d"; + }; + + void InitializeForInput(const InputType& i){ + m_table->InitializeForInput(i); + } + + Scores GetProb(const Phrase& f, const Phrase& e) const; + virtual FFState* EvaluateChart(const ChartHypothesis&, int /* featureID */, ScoreComponentCollection*) const { @@ -49,37 +57,20 @@ public: return NULL; } - virtual const FFState* EmptyHypothesisState(const InputType &input) const; - - virtual std::string GetScoreProducerDescription(unsigned) const { - return "LexicalReordering_" + m_modelTypeString; - } - - std::string GetScoreProducerWeightShortName(unsigned) const { - return "d"; - }; - - void InitializeForInput(const InputType& i) { - m_table->InitializeForInput(i); - } - - Scores GetProb(const Phrase& f, const Phrase& e) const; - private: - bool DecodeCondition(std::string s); - bool DecodeDirection(std::string s); - bool DecodeNumFeatureFunctions(std::string s); - - LexicalReorderingConfiguration m_configuration; - std::string m_modelTypeString; - std::vector<std::string> m_modelType; - LexicalReorderingTable* m_table; - size_t m_numScoreComponents; - //std::vector<Direction> m_direction; - std::vector<LexicalReorderingConfiguration::Condition> m_condition; - //std::vector<size_t> m_scoreOffset; - //bool m_oneScorePerDirection; - std::vector<FactorType> m_factorsE, m_factorsF; + bool DecodeCondition(std::string s); + bool DecodeDirection(std::string s); + bool DecodeNumFeatureFunctions(std::string s); + + LexicalReorderingConfiguration m_configuration; + std::string m_modelTypeString; + std::vector<std::string> m_modelType; + LexicalReorderingTable* m_table; + //std::vector<Direction> m_direction; + std::vector<LexicalReorderingConfiguration::Condition> m_condition; + //std::vector<size_t> m_scoreOffset; + //bool m_oneScorePerDirection; + std::vector<FactorType> m_factorsE, m_factorsF; }; } diff --git a/moses/src/LexicalReorderingState.cpp b/moses/src/LexicalReorderingState.cpp index 0ad0852df..4d9bcce07 100644 --- a/moses/src/LexicalReorderingState.cpp +++ b/moses/src/LexicalReorderingState.cpp @@ -33,14 +33,19 @@ size_t LexicalReorderingConfiguration::GetNumScoreComponents() const { size_t score_per_dir = m_collapseScores ? 1 : GetNumberOfTypes(); if (m_direction == Bidirectional) { - return 2 * score_per_dir; + return 2 * score_per_dir + m_additionalScoreComponents; } else { - return score_per_dir; + return score_per_dir + m_additionalScoreComponents; } } -LexicalReorderingConfiguration::LexicalReorderingConfiguration(ScoreProducer *scoreProducer, const std::string &modelType) - : m_scoreProducer(scoreProducer), m_modelType(None), m_phraseBased(true), m_collapseScores(false), m_direction(Backward) +void LexicalReorderingConfiguration::SetAdditionalScoreComponents(size_t number) +{ + m_additionalScoreComponents = number; +} + +LexicalReorderingConfiguration::LexicalReorderingConfiguration(const std::string &modelType) + : m_modelString(modelType), m_scoreProducer(NULL), m_modelType(None), m_phraseBased(true), m_collapseScores(false), m_direction(Backward), m_additionalScoreComponents(0) { std::vector<std::string> config = Tokenize<std::string>(modelType, "-"); diff --git a/moses/src/LexicalReorderingState.h b/moses/src/LexicalReorderingState.h index e4875c2a4..0eb4cc42e 100644 --- a/moses/src/LexicalReorderingState.h +++ b/moses/src/LexicalReorderingState.h @@ -16,6 +16,7 @@ namespace Moses { class LexicalReorderingState; +class LexicalReordering; /** Factory class for lexical reordering states * @todo There's a lot of classes for lexicalized reordering. Perhaps put them in a separate dir @@ -23,21 +24,24 @@ class LexicalReorderingState; class LexicalReorderingConfiguration { public: + friend class LexicalReordering; enum ModelType {Monotonic, MSD, MSLR, LeftRight, None}; enum Direction {Forward, Backward, Bidirectional}; enum Condition {F, E, FE}; - LexicalReorderingConfiguration(ScoreProducer *scoreProducer, const std::string &modelType); + LexicalReorderingConfiguration(const std::string &modelType); LexicalReorderingState *CreateLexicalReorderingState(const InputType &input) const; size_t GetNumScoreComponents() const; + void SetAdditionalScoreComponents(size_t number); size_t GetNumberOfTypes() const; ScoreProducer *GetScoreProducer() const { return m_scoreProducer; } + ModelType GetModelType() const { return m_modelType; } @@ -59,19 +63,28 @@ public: } private: + void SetScoreProducer(ScoreProducer* scoreProducer) { + m_scoreProducer = scoreProducer; + } + + const std::string& GetModelString() const { + return m_modelString; + } + + std::string m_modelString; ScoreProducer *m_scoreProducer; ModelType m_modelType; bool m_phraseBased; bool m_collapseScores; Direction m_direction; Condition m_condition; + size_t m_additionalScoreComponents; }; //! Abstract class for lexical reordering model states class LexicalReorderingState : public FFState { public: - virtual int Compare(const FFState& o) const = 0; virtual LexicalReorderingState* Expand(const TranslationOption& hypo, Scores& scores) const = 0; @@ -81,6 +94,7 @@ public: protected: typedef int ReorderingType; + const LexicalReorderingConfiguration &m_configuration; // The following is the true direction of the object, which can be Backward or Forward even if the Configuration has Bidirectional. LexicalReorderingConfiguration::Direction m_direction; diff --git a/moses/src/Makefile.am b/moses/src/Makefile.am new file mode 100644 index 000000000..b43191773 --- /dev/null +++ b/moses/src/Makefile.am @@ -0,0 +1,359 @@ +lib_LTLIBRARIES = libmoses.la +AM_CPPFLAGS = -W -Wall -ffor-scope -D_FILE_OFFSET_BITS=64 -D_LARGE_FILES $(BOOST_CPPFLAGS) + +libmoses_ladir = ${includedir} + +libmoses_la_HEADERS = \ + AlignmentInfo.h \ + AlignmentInfoCollection.h \ + BilingualDynSuffixArray.h \ + BitmapContainer.h \ + BleuScoreFeature.h \ + CellCollection.h \ + ChartCell.h \ + ChartCellCollection.h \ + ChartHypothesis.h \ + ChartHypothesisCollection.h \ + ChartManager.h \ + ChartRuleLookupManager.h \ + ChartRuleLookupManagerMemory.h \ + ChartRuleLookupManagerOnDisk.h \ + ChartTranslationOption.h \ + ChartTranslationOptionCollection.h \ + ChartTranslationOptionList.h \ + ChartTrellisDetour.h \ + ChartTrellisDetourQueue.h \ + ChartTrellisNode.h \ + ChartTrellisPath.h \ + ChartTrellisPathList.h \ + ConfusionNet.h \ + DecodeFeature.h \ + DecodeGraph.h \ + DecodeStep.h \ + DecodeStepGeneration.h \ + DecodeStepTranslation.h \ + Dictionary.h \ + DotChart.h \ + DotChartInMemory.h \ + DotChartOnDisk.h \ + DummyScoreProducers.h \ + DynSAInclude/file.h \ + DynSAInclude/vocab.h \ + DynSuffixArray.h \ + FFState.h \ + Factor.h \ + FactorCollection.h \ + FactorTypeSet.h \ + FeatureFunction.h \ + FeatureVector.h \ + File.h \ + FilePtr.h \ + FloydWarshall.h \ + GenerationDictionary.h \ + GlobalLexicalModel.h \ + GlobalLexicalModelUnlimited.h \ + gzfilebuf.h \ + hash.h \ + Hypothesis.h \ + HypothesisStack.h \ + HypothesisStackCubePruning.h \ + HypothesisStackNormal.h \ + InputFileStream.h \ + InputType.h \ + LMList.h \ + LVoc.h \ + LM/Base.h \ + LM/Joint.h \ + LM/Factory.h \ + LM/Implementation.h \ + LM/MultiFactor.h \ + LM/Remote.h \ + LM/SingleFactor.h \ + LM/Ken.h \ + LexicalReordering.h \ + LexicalReorderingState.h \ + LexicalReorderingTable.h \ + Manager.h \ + NonTerminal.h \ + ObjectPool.h \ + OnlineCommand.h \ + PCNTools.h \ + PDTAimp.h \ + Parameter.h \ + PartialTranslOptColl.h \ + Phrase.h \ + PhraseBoundaryFeature.h \ + PhraseDictionary.h \ + PhraseDictionaryALSuffixArray.h \ + PhraseDictionaryDynSuffixArray.h \ + PhraseDictionaryMemory.h \ + PhraseDictionarySCFG.h \ + PhraseDictionaryNode.h \ + PhraseDictionaryNodeSCFG.h \ + PhraseDictionaryOnDisk.h \ + PhraseDictionaryTree.h \ + PhraseDictionaryTreeAdaptor.h \ + PhraseLengthFeature.h \ + PhrasePairFeature.h \ + PrefixTree.h \ + PrefixTreeMap.h \ + ReorderingConstraint.h \ + ReorderingStack.h \ + RuleCube.h \ + RuleCubeItem.h \ + RuleCubeQueue.h \ + RuleTableLoader.h \ + RuleTableLoaderCompact.h \ + RuleTableLoaderFactory.h \ + RuleTableLoaderHiero.h \ + RuleTableLoaderStandard.h \ + ScoreComponentCollection.h \ + ScoreProducer.h \ + Search.h \ + SearchCubePruning.h \ + SearchNormal.h \ + Sentence.h \ + SentenceStats.h \ + SourceWordDeletionFeature.h \ + SparsePhraseDictionaryFeature.h \ + SquareMatrix.h \ + StaticData.h \ + TargetBigramFeature.h \ + TargetNgramFeature.h \ + TargetPhrase.h \ + TargetPhraseCollection.h \ + TargetWordInsertionFeature.h \ + ThreadPool.h \ + Timer.h \ + TranslationOption.h \ + TranslationOptionCollection.h \ + TranslationOptionCollectionConfusionNet.h \ + TranslationOptionCollectionText.h \ + TranslationOptionList.h \ + TranslationSystem.h \ + TreeInput.h \ + TrellisPath.h \ + TrellisPathCollection.h \ + TrellisPathList.h \ + TypeDef.h \ + UniqueObject.h \ + UserMessage.h \ + Util.h \ + Word.h \ + WordLattice.h \ + WordTranslationFeature.h \ + WordsBitmap.h \ + WordsRange.h \ + XmlOption.h + +if PROTOBUF +libmoses_la_HEADERS += rule.pb.h hypergraph.pb.h +endif + +if SRI_LM +libmoses_la_HEADERS += LM/SRI.h \ + LM/ParallelBackoff.h +endif + +if IRST_LM +libmoses_la_HEADERS += LM/IRST.h +endif + +if RAND_LM +libmoses_la_HEADERS += LM/RandLM.h +endif + +if ORLM_LM +libmoses_la_HEADERS += LM/ORLM.h \ + DynSAInclude/params.h \ + DynSAInclude/hash.h \ + DynSAInclude/quantizer.h \ + DynSAInclude/RandLMFilter.h \ + DynSAInclude/RandLMCache.h +endif + +if SYN_LM +libmoses_la_HEADERS += SyntacticLanguageModel.h +endif + +libmoses_la_SOURCES = \ + AlignmentInfo.cpp \ + AlignmentInfoCollection.cpp \ + BilingualDynSuffixArray.cpp \ + BitmapContainer.cpp \ + BleuScoreFeature.cpp \ + ChartCell.cpp \ + ChartCellCollection.cpp \ + ChartHypothesis.cpp \ + ChartHypothesisCollection.cpp \ + ChartManager.cpp \ + ChartRuleLookupManager.cpp \ + ChartRuleLookupManagerMemory.cpp \ + ChartRuleLookupManagerOnDisk.cpp \ + ChartTranslationOption.cpp \ + ChartTranslationOptionCollection.cpp \ + ChartTranslationOptionList.cpp \ + ChartTrellisDetour.cpp \ + ChartTrellisDetourQueue.cpp \ + ChartTrellisNode.cpp \ + ChartTrellisPath.cpp \ + ConfusionNet.cpp \ + DecodeFeature.cpp \ + DecodeGraph.cpp \ + DecodeStep.cpp \ + DecodeStepGeneration.cpp \ + DecodeStepTranslation.cpp \ + Dictionary.cpp \ + DotChart.cpp \ + DotChartInMemory.cpp \ + DotChartOnDisk.cpp \ + DummyScoreProducers.cpp \ + DynSAInclude/file.cpp \ + DynSAInclude/vocab.cpp \ + DynSuffixArray.cpp \ + FFState.cpp \ + Factor.cpp \ + FactorCollection.cpp \ + FactorTypeSet.cpp \ + FeatureFunction.cpp \ + FeatureVector.cpp \ + FloydWarshall.cpp \ + GenerationDictionary.cpp \ + GlobalLexicalModel.cpp \ + GlobalLexicalModelUnlimited.cpp \ + hash.cpp \ + Hypothesis.cpp \ + HypothesisStack.cpp \ + HypothesisStackCubePruning.cpp \ + HypothesisStackNormal.cpp \ + InputFileStream.cpp \ + InputType.cpp \ + LMList.cpp \ + LVoc.cpp \ + LM/Base.cpp \ + LM/Factory.cpp \ + LM/Implementation.cpp \ + LM/Joint.cpp \ + LM/Ken.cpp \ + LM/MultiFactor.cpp \ + LM/Remote.cpp \ + LM/SingleFactor.cpp \ + LexicalReordering.cpp \ + LexicalReorderingState.cpp \ + LexicalReorderingTable.cpp \ + Manager.cpp \ + OnlineCommand.cpp \ + PCNTools.cpp \ + Parameter.cpp \ + PartialTranslOptColl.cpp \ + Phrase.cpp \ + PhraseBoundaryFeature.cpp \ + PhraseDictionary.cpp \ + PhraseDictionaryALSuffixArray.cpp \ + PhraseDictionaryDynSuffixArray.cpp \ + PhraseDictionaryHiero.cpp \ + PhraseDictionaryMemory.cpp \ + PhraseDictionarySCFG.cpp \ + PhraseDictionaryNode.cpp \ + PhraseDictionaryNodeSCFG.cpp \ + PhraseDictionaryOnDisk.cpp \ + PhraseDictionaryTree.cpp \ + PhraseDictionaryTreeAdaptor.cpp \ + PhraseLengthFeature.cpp \ + PhrasePairFeature.cpp \ + PrefixTreeMap.cpp \ + ReorderingConstraint.cpp \ + ReorderingStack.cpp \ + RuleCube.cpp \ + RuleCubeItem.cpp \ + RuleCubeQueue.cpp \ + RuleTableLoaderCompact.cpp \ + RuleTableLoaderFactory.cpp \ + RuleTableLoaderHiero.cpp \ + RuleTableLoaderStandard.cpp \ + ScoreComponentCollection.cpp \ + ScoreProducer.cpp \ + Search.cpp \ + SearchCubePruning.cpp \ + SearchNormal.cpp \ + Sentence.cpp \ + SentenceStats.cpp \ + SourceWordDeletionFeature.cpp \ + SparsePhraseDictionaryFeature.cpp \ + SquareMatrix.cpp \ + StaticData.cpp \ + TargetBigramFeature.cpp \ + TargetNgramFeature.cpp \ + TargetPhrase.cpp \ + TargetPhraseCollection.cpp \ + TargetWordInsertionFeature.cpp \ + ThreadPool.cpp \ + Timer.cpp \ + TranslationOption.cpp \ + TranslationOptionCollection.cpp \ + TranslationOptionCollectionConfusionNet.cpp \ + TranslationOptionCollectionText.cpp \ + TranslationOptionList.cpp \ + TranslationSystem.cpp \ + TreeInput.cpp \ + TrellisPath.cpp \ + TrellisPathCollection.cpp \ + UserMessage.cpp \ + Util.cpp \ + Word.cpp \ + WordLattice.cpp \ + WordTranslationFeature.cpp \ + WordsBitmap.cpp \ + WordsRange.cpp \ + XmlOption.cpp + +if PROTOBUF +BUILT_SOURCES = \ + rule.pb.h \ + rule.pb.cc \ + hypergraph.pb.h \ + hypergraph.pb.cc + +CLEANFILES = $(BUILT_SOURCES) +SUFFIXES = .proto + +rule.pb.cc: rule.proto + @PROTOC@ --cpp_out=. $< +rule.pb.h: rule.proto + @PROTOC@ --cpp_out=. $< + +hypergraph.pb.cc: hypergraph.proto + @PROTOC@ --cpp_out=. $< +hypergraph.pb.h: hypergraph.proto + @PROTOC@ --cpp_out=. $< + +libmoses_la_SOURCES += rule.pb.cc hypergraph.pb.cc + +endif + +if SRI_LM +libmoses_la_SOURCES += LM/SRI.cpp \ + LM/ParallelBackoff.cpp + +endif + +if IRST_LM +libmoses_la_SOURCES += LM/IRST.cpp +endif + +if RAND_LM +libmoses_la_SOURCES += LM/RandLM.cpp +endif + +if ORLM_LM +libmoses_la_SOURCES += LM/ORLM.cpp \ + DynSAInclude/onlineRLM.h \ + DynSAInclude/perfecthash.h \ + DynSAInclude/params.cpp +endif + +if SYN_LM +libmoses_la_SOURCES += SyntacticLanguageModel.cpp +endif + +libmoses_la_LIBADD = $(top_srcdir)/util/libkenutil.la $(top_srcdir)/lm/libkenlm.la $(BOOST_THREAD_LDFLAGS) $(BOOST_THREAD_LIBS) diff --git a/moses/src/Manager.cpp b/moses/src/Manager.cpp index fc96cfaab..468db0de3 100644 --- a/moses/src/Manager.cpp +++ b/moses/src/Manager.cpp @@ -638,7 +638,7 @@ void OutputSearchNode(long translationId, std::ostream &outputSearchGraphStream, // special case: initial hypothesis if ( searchNode.hypo->GetId() == 0 ) { outputSearchGraphStream << " hyp=0 stack=0"; - if (!extendedFormat) { + if (extendedFormat) { outputSearchGraphStream << " forward=" << searchNode.forward << " fscore=" << searchNode.fscore; } outputSearchGraphStream << endl; @@ -667,33 +667,31 @@ void OutputSearchNode(long translationId, std::ostream &outputSearchGraphStream, } // output in extended format - if (searchNode.recombinationHypo != NULL) - outputSearchGraphStream << " hyp=" << searchNode.recombinationHypo->GetId(); - else - outputSearchGraphStream << " hyp=" << searchNode.hypo->GetId(); +// if (searchNode.recombinationHypo != NULL) +// outputSearchGraphStream << " hyp=" << searchNode.recombinationHypo->GetId(); +// else + outputSearchGraphStream << " hyp=" << searchNode.hypo->GetId(); - outputSearchGraphStream << " stack=" << searchNode.hypo->GetWordsBitmap().GetNumWordsCovered() - << " back=" << prevHypo->GetId() - << " score=" << searchNode.hypo->GetScore() - << " transition=" << (searchNode.hypo->GetScore() - prevHypo->GetScore()); + outputSearchGraphStream << " stack=" << searchNode.hypo->GetWordsBitmap().GetNumWordsCovered() + << " back=" << prevHypo->GetId() + << " score=" << searchNode.hypo->GetScore() + << " transition=" << (searchNode.hypo->GetScore() - prevHypo->GetScore()); - if (searchNode.recombinationHypo != NULL) - outputSearchGraphStream << " recombined=" << searchNode.recombinationHypo->GetId(); + if (searchNode.recombinationHypo != NULL) + outputSearchGraphStream << " recombined=" << searchNode.recombinationHypo->GetId(); - outputSearchGraphStream << " forward=" << searchNode.forward << " fscore=" << searchNode.fscore + outputSearchGraphStream << " forward=" << searchNode.forward << " fscore=" << searchNode.fscore << " covered=" << searchNode.hypo->GetCurrSourceWordsRange().GetStartPos() << "-" << searchNode.hypo->GetCurrSourceWordsRange().GetEndPos(); // Modified so that -osgx is a superset of -osg (GST Oct 2011) ScoreComponentCollection scoreBreakdown = searchNode.hypo->GetScoreBreakdown(); scoreBreakdown.MinusEquals( prevHypo->GetScoreBreakdown() ); - outputSearchGraphStream << " scores=[ "; - StaticData::Instance().GetScoreIndexManager().PrintLabeledScores( outputSearchGraphStream, scoreBreakdown ); - outputSearchGraphStream << " ]"; - + //outputSearchGraphStream << " scores = [ " << StaticData::Instance().GetAllWeights(); + outputSearchGraphStream << " scores=\"" << scoreBreakdown << "\""; - outputSearchGraphStream << " out=" << searchNode.hypo->GetSourcePhraseStringRep() << "|" << - searchNode.hypo->GetCurrTargetPhrase().GetStringRep(outputFactorOrder) << endl; + outputSearchGraphStream << " out=\"" << searchNode.hypo->GetSourcePhraseStringRep() << "|" << + searchNode.hypo->GetCurrTargetPhrase().GetStringRep(outputFactorOrder) << "\"" << endl; // outputSearchGraphStream << " out=" << searchNode.hypo->GetCurrTargetPhrase().GetStringRep(outputFactorOrder) << endl; } diff --git a/moses/src/MockHypothesis.cpp b/moses/src/MockHypothesis.cpp new file mode 100644 index 000000000..0afd3b071 --- /dev/null +++ b/moses/src/MockHypothesis.cpp @@ -0,0 +1,102 @@ +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2010 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + + + +#include "MockHypothesis.h" + +#include <boost/test/unit_test.hpp> + +#include "TranslationOption.h" + +using namespace Moses; +using namespace std; + +namespace MosesTest { + + +MockHypothesisGuard::MockHypothesisGuard( + const string& sourceSentence, + const vector<Alignment>& alignments, + const vector<string>& targetSegments) +: m_emptyTarget(), + m_sentence(), + m_system("mock",&m_wp,&m_uwp,&m_dist), + m_manager(m_sentence,Normal,&m_system) +{ + BOOST_CHECK_EQUAL(alignments.size(), targetSegments.size()); + + std::vector<Moses::FactorType> factors; + factors.push_back(0); + + stringstream in(sourceSentence + "\n"); + m_sentence.Read(in,factors); + + + //Initial empty hypothesis + m_manager.ResetSentenceStats(m_sentence); + m_hypothesis = Hypothesis::Create(m_manager, m_sentence, m_emptyTarget); + + //create the chain + vector<Alignment>::const_iterator ai = alignments.begin(); + vector<string>::const_iterator ti = targetSegments.begin(); + for (; ti != targetSegments.end() && ai != alignments.end(); ++ti,++ai) + { + Hypothesis* prevHypo = m_hypothesis; + WordsRange wordsRange(ai->first,ai->second); + m_targetPhrases.push_back(TargetPhrase()); + m_targetPhrases.back().CreateFromString(factors,*ti,"|"); + m_toptions.push_back(new TranslationOption + (wordsRange,m_targetPhrases.back(),m_sentence)); + m_hypothesis = Hypothesis::Create(*prevHypo,*m_toptions.back(),NULL); + } + + +} + +MockHypothesisGuard::~MockHypothesisGuard() +{ + RemoveAllInColl(m_toptions); + while (m_hypothesis) { + Hypothesis* prevHypo = const_cast<Hypothesis*>(m_hypothesis->GetPrevHypo()); + delete m_hypothesis; + m_hypothesis = prevHypo; + } +} + +HypothesisFixture::HypothesisFixture() +{ + string source = "je ne sais pas . "; + vector<string> target; + vector<Alignment> alignments; + m_empty.reset(new MockHypothesisGuard(source,alignments,target)); + target.push_back("i"); + target.push_back("do not"); + alignments.push_back(Alignment(0,0)); + alignments.push_back(Alignment(3,3)); + m_partial.reset(new MockHypothesisGuard(source,alignments,target)); + target.push_back("know"); + target.push_back("."); + alignments.push_back(Alignment(1,2)); + alignments.push_back(Alignment(4,4)); + m_full.reset(new MockHypothesisGuard(source,alignments,target)); +} + +} + diff --git a/moses/src/MockHypothesis.h b/moses/src/MockHypothesis.h new file mode 100644 index 000000000..41f118f40 --- /dev/null +++ b/moses/src/MockHypothesis.h @@ -0,0 +1,83 @@ +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2010 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#ifndef _MOCK_HYPOTHESIS_ +#define _MOCK_HYPOTHESIS_ + +#include <memory> +#include <vector> + + +#include "DummyScoreProducers.h" +#include "Hypothesis.h" +#include "Manager.h" +#include "TranslationSystem.h" + +namespace MosesTest { + +// +// Construct a hypothesis with arbitrary source and target phrase +// sequences. Useful for testing feature functions. +// + +typedef std::pair<size_t,size_t> Alignment; //(first,last) in source + +class MockHypothesisGuard { + public: + /** Creates a phrase-based hypothesis. + */ + MockHypothesisGuard( + const std::string& sourceSentence, + const std::vector<Alignment>& alignments, + const std::vector<std::string>& targetSegments); + Moses::Hypothesis* operator*() const {return m_hypothesis;} + + /** Destroy the hypothesis chain */ + ~MockHypothesisGuard(); + + private: + Moses::TargetPhrase m_emptyTarget; + Moses::Sentence m_sentence; + Moses::WordPenaltyProducer m_wp; + Moses::UnknownWordPenaltyProducer m_uwp; + Moses::DistortionScoreProducer m_dist; + Moses::TranslationSystem m_system; + Moses::Manager m_manager; + Moses::Hypothesis* m_hypothesis; + std::vector<Moses::TargetPhrase> m_targetPhrases; + std::vector<Moses::TranslationOption*> m_toptions; +}; + +class HypothesisFixture { + public: + HypothesisFixture(); + const Moses::Hypothesis* empty() {return **m_empty;} + const Moses::Hypothesis* partial() {return **m_partial;} + const Moses::Hypothesis* full() {return **m_full;} + + private: + std::auto_ptr<MockHypothesisGuard> m_empty; + std::auto_ptr<MockHypothesisGuard> m_partial; + std::auto_ptr<MockHypothesisGuard> m_full; +}; + + +} + +#endif diff --git a/moses/src/MosesTest.cpp b/moses/src/MosesTest.cpp new file mode 100644 index 000000000..45f5bdba4 --- /dev/null +++ b/moses/src/MosesTest.cpp @@ -0,0 +1,24 @@ +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2010 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + + +//Supplies the main for the moses test module +#define BOOST_TEST_MODULE moses +#include <boost/test/unit_test.hpp> + diff --git a/moses/src/PDTAimp.h b/moses/src/PDTAimp.h index 96a64cad6..2d2b6974c 100644 --- a/moses/src/PDTAimp.h +++ b/moses/src/PDTAimp.h @@ -9,6 +9,7 @@ #include "UniqueObject.h" #include "InputFileStream.h" #include "PhraseDictionaryTreeAdaptor.h" +#include "SparsePhraseDictionaryFeature.h" #include "Util.h" namespace Moses @@ -34,13 +35,11 @@ class PDTAimp protected: PDTAimp(PhraseDictionaryTreeAdaptor *p,unsigned nis) - : m_languageModels(0),m_weightWP(0.0),m_dict(0), + : m_languageModels(0),m_dict(0), m_obj(p),useCache(1),m_numInputScores(nis),totalE(0),distinctE(0) {} -public: - std::vector<float> m_weights; + public: LMList const* m_languageModels; - float m_weightWP; std::vector<FactorType> m_input,m_output; PhraseDictionaryTree *m_dict; typedef std::vector<TargetPhraseCollection const*> vTPC; @@ -140,6 +139,11 @@ public: return 0; } + //TODO: Multiple models broken here + const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT); + std::vector<float> weights = StaticData::Instance().GetWeights(m_obj->GetFeature()); + float weightWP = system.GetWeightWordPenalty(); + std::vector<TargetPhrase> tCands; tCands.reserve(cands.size()); std::vector<std::pair<float,size_t> > costs; @@ -149,16 +153,24 @@ public: for(size_t i=0; i<cands.size(); ++i) { TargetPhrase targetPhrase(Output); - StringTgtCand::first_type const& factorStrings=cands[i].first; - StringTgtCand::second_type const& probVector=cands[i].second; + StringTgtCand::Tokens const& factorStrings=cands[i].tokens; + Scores const& probVector=cands[i].scores; std::vector<float> scoreVector(probVector.size()); std::transform(probVector.begin(),probVector.end(),scoreVector.begin(), TransformScore); std::transform(scoreVector.begin(),scoreVector.end(),scoreVector.begin(), FloorScore); - //CreateTargetPhrase(targetPhrase,factorStrings,scoreVector,&src); - CreateTargetPhrase(targetPhrase,factorStrings,scoreVector,wacands[i],&src); + //sparse features. + //These are already in log-space + ScoreComponentCollection sparseFeatures; + if (m_obj->GetFeature()->GetSparsePhraseDictionaryFeature()) { + for (size_t j = 0; j < cands[i].fnames.size(); ++j) { + sparseFeatures.Assign(m_obj->GetFeature()->GetSparsePhraseDictionaryFeature(), + *(cands[i].fnames[j]), cands[i].fvalues[j]); + } + } + CreateTargetPhrase(targetPhrase,factorStrings,scoreVector, sparseFeatures, wacands[i], weights, weightWP, &src); costs.push_back(std::make_pair(-targetPhrase.GetFutureScore(),tCands.size())); tCands.push_back(targetPhrase); } @@ -181,9 +193,8 @@ public: void Create(const std::vector<FactorType> &input , const std::vector<FactorType> &output , const std::string &filePath - , const std::vector<float> &weight + , const std::vector<float> &weight , const LMList &languageModels - , float weightWP ) { // set my members @@ -191,8 +202,6 @@ public: m_input=input; m_output=output; m_languageModels=&languageModels; - m_weightWP=weightWP; - m_weights=weight; const StaticData &staticData = StaticData::Instance(); m_dict->UseWordAlignment(staticData.UseAlignmentInfo()); @@ -253,18 +262,24 @@ public: }; void CreateTargetPhrase(TargetPhrase& targetPhrase, - StringTgtCand::first_type const& factorStrings, - StringTgtCand::second_type const& scoreVector, + StringTgtCand::Tokens const& factorStrings, + Scores const& scoreVector, + const ScoreComponentCollection& sparseFeatures, const std::string& alignmentString, + std::vector<float> &weights, + float weightWP, Phrase const* srcPtr=0) const { - CreateTargetPhrase(targetPhrase, factorStrings, scoreVector, srcPtr); + CreateTargetPhrase(targetPhrase, factorStrings, scoreVector, sparseFeatures, weights, weightWP, srcPtr); targetPhrase.SetAlignmentInfo(alignmentString); } void CreateTargetPhrase(TargetPhrase& targetPhrase, - StringTgtCand::first_type const& factorStrings, - StringTgtCand::second_type const& scoreVector, + StringTgtCand::Tokens const& factorStrings, + Scores const& scoreVector, + const ScoreComponentCollection& sparseFeatures, + std::vector<float> &weights, + float weightWP, Phrase const* srcPtr=0) const { FactorCollection &factorCollection = FactorCollection::Instance(); @@ -276,8 +291,9 @@ public: w[m_output[l]]= factorCollection.AddFactor(Output, m_output[l], factors[l]); } } - targetPhrase.SetScore(m_obj->GetFeature(), scoreVector, m_weights, m_weightWP, *m_languageModels); - targetPhrase.SetSourcePhrase(srcPtr); + + targetPhrase.SetScore(m_obj->GetFeature(), scoreVector, sparseFeatures, weights, weightWP, *m_languageModels); + targetPhrase.SetSourcePhrase(*srcPtr); } @@ -308,7 +324,7 @@ public: // POD for target phrase scores struct TScores { float total; - StringTgtCand::second_type trans; + Scores trans; Phrase const* src; TScores() : total(0.0),src(0) {} @@ -350,14 +366,18 @@ public: TRACE_ERR("\n"); } - typedef StringTgtCand::first_type sPhrase; - typedef std::map<StringTgtCand::first_type,TScores> E2Costs; + typedef StringTgtCand::Tokens sPhrase; + typedef std::map<StringTgtCand::Tokens,TScores> E2Costs; std::map<Range,E2Costs> cov2cand; std::vector<State> stack; for(Position i=0 ; i < srcSize ; ++i) stack.push_back(State(i, i, m_dict->GetRoot(), std::vector<float>(m_numInputScores,0.0))); + const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT); + std::vector<float> weightT = StaticData::Instance().GetWeights(m_obj->GetFeature()); + float weightWP = system.GetWeightWordPenalty(); + while(!stack.empty()) { State curr(stack.back()); stack.pop_back(); @@ -428,20 +448,20 @@ public: std::vector<float> nscores(newInputScores); //resize to include phrase table scores - nscores.resize(m_numInputScores+tcands[i].second.size(),0.0f); + nscores.resize(m_numInputScores+tcands[i].scores.size(),0.0f); //put in phrase table scores, logging as we insert - std::transform(tcands[i].second.begin(),tcands[i].second.end(),nscores.begin() + m_numInputScores,TransformScore); - - CHECK(nscores.size()==m_weights.size()); + std::transform(tcands[i].scores.begin(),tcands[i].scores.end(),nscores.begin() + m_numInputScores,TransformScore); + + CHECK(nscores.size()==weightT.size()); //tally up - float score=std::inner_product(nscores.begin(), nscores.end(), m_weights.begin(), 0.0f); + float score=std::inner_product(nscores.begin(), nscores.end(), weightT.begin(), 0.0f); //count word penalty - score-=tcands[i].first.size() * m_weightWP; + score-=tcands[i].tokens.size() * weightWP; - std::pair<E2Costs::iterator,bool> p=e2costs.insert(std::make_pair(tcands[i].first,TScores())); + std::pair<E2Costs::iterator,bool> p=e2costs.insert(std::make_pair(tcands[i].tokens,TScores())); if(p.second) ++distinctE; @@ -487,7 +507,7 @@ public: for(E2Costs::const_iterator j=i->second.begin(); j!=i->second.end(); ++j) { TScores const & scores=j->second; TargetPhrase targetPhrase(Output); - CreateTargetPhrase(targetPhrase,j->first,scores.trans,scores.src); + CreateTargetPhrase(targetPhrase,j->first,scores.trans,ScoreComponentCollection(),weightT,weightWP,scores.src); costs.push_back(std::make_pair(-targetPhrase.GetFutureScore(),tCands.size())); tCands.push_back(targetPhrase); //std::cerr << i->first.first << "-" << i->first.second << ": " << targetPhrase << std::endl; diff --git a/moses/src/Parameter.cpp b/moses/src/Parameter.cpp index aab5dbc06..230a9ddb5 100644 --- a/moses/src/Parameter.cpp +++ b/moses/src/Parameter.cpp @@ -28,6 +28,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "Parameter.h" #include "Util.h" #include "InputFileStream.h" +#include "StaticData.h" #include "UserMessage.h" using namespace std; @@ -41,11 +42,14 @@ Parameter::Parameter() AddParam("config", "f", "location of the configuration file"); AddParam("continue-partial-translation", "cpt", "start from nonempty hypothesis"); AddParam("decoding-graph-backoff", "dpb", "only use subsequent decoding paths for unknown spans of given length"); + AddParam("dlm-model", "Order, factor and vocabulary file for discriminative LM. Use * for filename to indicate unlimited vocabulary."); AddParam("drop-unknown", "du", "drop unknown words instead of copying them"); AddParam("disable-discarding", "dd", "disable hypothesis discarding"); + AddParam("distinct-nbest", "only distinct translations in nbest list"); AddParam("factor-delimiter", "fd", "specify a different factor delimiter than the default"); AddParam("generation-file", "location and properties of the generation table"); AddParam("global-lexical-file", "gl", "discriminatively trained global lexical translation model file"); + AddParam("glm-feature", "discriminatively trained global lexical translation feature, sparse producer"); AddParam("input-factors", "list of factors in the input"); AddParam("input-file", "i", "location of the input file to be translated"); AddParam("inputtype", "text (0), confusion network (1), word lattice (2) (default = 0)"); @@ -76,33 +80,41 @@ Parameter::Parameter() AddParam("stack", "s", "maximum stack size for histogram pruning"); AddParam("stack-diversity", "sd", "minimum number of hypothesis of each coverage in stack (default 0)"); AddParam("threads","th", "number of threads to use in decoding (defaults to single-threaded)"); - AddParam("translation-details", "T", "for each best hypothesis, report translation details to the given file"); - AddParam("ttable-file", "location and properties of the translation tables"); - AddParam("ttable-limit", "ttl", "maximum number of translation table entries per input phrase"); - AddParam("translation-option-threshold", "tot", "threshold for translation options relative to best for input phrase"); - AddParam("early-discarding-threshold", "edt", "threshold for constructing hypotheses based on estimate cost"); - AddParam("verbose", "v", "verbosity level of the logging"); - AddParam("weight-d", "d", "weight(s) for distortion (reordering components)"); + AddParam("translation-details", "T", "for each best hypothesis, report translation details to the given file"); + AddParam("ttable-file", "location and properties of the translation tables"); + AddParam("ttable-limit", "ttl", "maximum number of translation table entries per input phrase"); + AddParam("translation-option-threshold", "tot", "threshold for translation options relative to best for input phrase"); + AddParam("early-discarding-threshold", "edt", "threshold for constructing hypotheses based on estimate cost"); + AddParam("verbose", "v", "verbosity level of the logging"); + AddParam("references", "Reference file(s) - used for bleu score feature"); + AddParam("weight-bl", "bl", "weight for bleu score feature"); + AddParam("weight-d", "d", "weight(s) for distortion (reordering components)"); + AddParam("weight-dlm", "dlm", "weight for discriminative LM feature function (on top of sparse weights)"); AddParam("weight-lr", "lr", "weight(s) for lexicalized reordering, if not included in weight-d"); - AddParam("weight-generation", "g", "weight(s) for generation components"); - AddParam("weight-i", "I", "weight(s) for word insertion - used for parameters from confusion network and lattice input links"); - AddParam("weight-l", "lm", "weight(s) for language models"); - AddParam("weight-lex", "lex", "weight for global lexical model"); - AddParam("weight-t", "tm", "weights for translation model components"); - AddParam("weight-w", "w", "weight for word penalty"); - AddParam("weight-u", "u", "weight for unknown word penalty"); - AddParam("weight-e", "e", "weight for word deletion"); - AddParam("output-factors", "list if factors in the output"); - AddParam("cache-path", "?"); - AddParam("distortion-limit", "dl", "distortion (reordering) limit in maximum number of words (0 = monotone, -1 = unlimited)"); - AddParam("monotone-at-punctuation", "mp", "do not reorder over punctuation"); - AddParam("distortion-file", "source factors (0 if table independent of source), target factors, location of the factorized/lexicalized reordering tables"); - AddParam("distortion", "configurations for each factorized/lexicalized reordering model."); - AddParam("early-distortion-cost", "edc", "include estimate of distortion cost yet to be incurred in the score [Moore & Quirk 2007]. Default is no"); - AddParam("xml-input", "xi", "allows markup of input with desired translations and probabilities. values can be 'pass-through' (default), 'inclusive', 'exclusive', 'ignore'"); + AddParam("weight-generation", "g", "weight(s) for generation components"); + AddParam("weight-i", "I", "weight(s) for word insertion - used for parameters from confusion network and lattice input links"); + AddParam("weight-l", "lm", "weight(s) for language models"); + AddParam("weight-lex", "lex", "weight for global lexical model"); + AddParam("weight-glm", "glm", "weight for global lexical feature, sparse producer"); + AddParam("weight-wt", "wt", "weight for word translation feature"); + AddParam("weight-pp", "pp", "weight for phrase pair feature"); + AddParam("weight-pb", "pb", "weight for phrase boundary feature"); + AddParam("weight-t", "tm", "weights for translation model components"); + AddParam("weight-w", "w", "weight for word penalty"); + AddParam("weight-u", "u", "weight for unknown word penalty"); + AddParam("weight-e", "e", "weight for word deletion"); + AddParam("weight-file", "wf", "feature weights file. Do *not* put weights for 'core' features in here - they go in moses.ini"); + AddParam("output-factors", "list if factors in the output"); + AddParam("cache-path", "?"); + AddParam("distortion-limit", "dl", "distortion (reordering) limit in maximum number of words (0 = monotone, -1 = unlimited)"); + AddParam("monotone-at-punctuation", "mp", "do not reorder over punctuation"); + AddParam("distortion-file", "source factors (0 if table independent of source), target factors, location of the factorized/lexicalized reordering tables"); + AddParam("distortion", "configurations for each factorized/lexicalized reordering model."); + AddParam("xml-input", "xi", "allows markup of input with desired translations and probabilities. values can be 'pass-through' (default), 'inclusive', 'exclusive', 'ignore'"); AddParam("xml-brackets", "xb", "specify strings to be used as xml tags opening and closing, e.g. \"{{ }}\" (default \"< >\"). Avoid square brackets because of configuration file format. Valid only with text input mode" ); - AddParam("minimum-bayes-risk", "mbr", "use miminum Bayes risk to determine best translation"); + AddParam("minimum-bayes-risk", "mbr", "use miminum Bayes risk to determine best translation"); AddParam("lminimum-bayes-risk", "lmbr", "use lattice miminum Bayes risk to determine best translation"); + AddParam("mira", "do mira training"); AddParam("consensus-decoding", "con", "use consensus decoding (De Nero et. al. 2009)"); AddParam("mbr-size", "number of translation candidates considered in MBR decoding (default 200)"); AddParam("mbr-scale", "scaling factor to convert log linear score probability in MBR decoding (default 1.0)"); @@ -125,8 +137,30 @@ Parameter::Parameter() #ifdef HAVE_PROTOBUF AddParam("output-search-graph-pb", "pb", "Write phrase lattice to protocol buffer objects in the specified path."); #endif - AddParam("cube-pruning-pop-limit", "cbp", "How many hypotheses should be popped for each stack. (default = 1000)"); - AddParam("cube-pruning-diversity", "cbd", "How many hypotheses should be created for each coverage. (default = 0)"); + AddParam("cube-pruning-pop-limit", "cbp", "How many hypotheses should be popped for each stack. (default = 1000)"); + AddParam("cube-pruning-diversity", "cbd", "How many hypotheses should be created for each coverage. (default = 0)"); + AddParam("search-algorithm", "Which search algorithm to use. 0=normal stack, 1=cube pruning, 2=cube growing. (default = 0)"); + AddParam("constraint", "Location of the file with target sentences to produce constraining the search"); + AddParam("use-alignment-info", "Use word-to-word alignment: actually it is only used to output the word-to-word alignment. Word-to-word alignments are taken from the phrase table if any. Default is false."); + AddParam("print-alignment-info", "Output word-to-word alignment into the log file. Word-to-word alignments are taken from the phrase table if any. Default is false"); + AddParam("print-alignment-info-in-n-best", "Include word-to-word alignment in the n-best list. Word-to-word alignments are takne from the phrase table if any. Default is false"); + AddParam("link-param-count", "Number of parameters on word links when using confusion networks or lattices (default = 1)"); + AddParam("description", "Source language, target language, description"); + AddParam("max-chart-span", "maximum num. of source word chart rules can consume (default 10)"); + AddParam("non-terminals", "list of non-term symbols, space separated"); + AddParam("rule-limit", "a little like table limit. But for chart decoding rules. Default is DEFAULT_MAX_TRANS_OPT_SIZE"); + AddParam("source-label-overlap", "What happens if a span already has a label. 0=add more. 1=replace. 2=discard. Default is 0"); + AddParam("output-hypo-score", "Output the hypo score to stdout with the output string. For search error analysis. Default is false"); + AddParam("unknown-lhs", "file containing target lhs of unknown words. 1 per line: LHS prob"); + AddParam("enable-online-command", "enable online commands to change some decoder parameters (default false); if enabled, use-persistent-cache is disabled"); + AddParam("phrase-pair-feature", "Source and target factors for phrase pair feature"); + AddParam("phrase-boundary-source-feature", "Source factors for phrase boundary feature"); + AddParam("phrase-boundary-target-feature", "Target factors for phrase boundary feature"); + AddParam("phrase-length-feature", "Count features for source length, target length, both of each phrase"); + AddParam("target-word-insertion-feature", "Count feature for each unaligned target word"); + AddParam("source-word-deletion-feature", "Count feature for each unaligned source word"); + AddParam("word-translation-feature", "Count feature for word translation according to word alignment"); + AddParam("report-sparse-features", "Indicate which sparse feature functions should report detailed scores in n-best, instead of aggregate"); AddParam("cube-pruning-lazy-scoring", "cbls", "Don't fully score a hypothesis until it is popped"); AddParam("parsing-algorithm", "Which parsing algorithm to use. 0=CYK+, 1=scope-3. (default = 0)"); AddParam("search-algorithm", "Which search algorithm to use. 0=normal stack, 1=cube pruning, 2=cube growing, 4=stack with batched lm requests (default = 0)"); @@ -148,6 +182,7 @@ Parameter::Parameter() AddParam("alignment-output-file", "print output word alignments into given file"); AddParam("sort-word-alignment", "Sort word alignments for more consistent display. 0=no sort (default), 1=target order"); AddParam("start-translation-id", "Id of 1st input. Default = 0"); + AddParam("text-type", "should be one of dev/devtest/test, used for domain adaptation features"); AddParam("output-unknowns", "Output the unknown (OOV) words to the given file, one line per sentence"); // Compact phrase table and reordering table. @@ -172,6 +207,7 @@ void Parameter::AddParam(const string ¶mName, const string &abbrevName, cons m_valid[paramName] = true; m_valid[abbrevName] = true; m_abbreviation[paramName] = abbrevName; + m_fullname[abbrevName] = paramName; m_description[paramName] = description; } @@ -606,6 +642,30 @@ void Parameter::PrintCredit() cerr << endl << endl; } +/** update parameter settings with command line switches + * \param paramName full name of parameter + * \param values inew values for paramName */ +void Parameter::OverwriteParam(const string ¶mName, PARAM_VEC values) +{ + VERBOSE(2,"Overwriting parameter " << paramName); + + m_setting[paramName]; // defines the parameter, important for boolean switches + if (m_setting[paramName].size() > 1){ + VERBOSE(2," (the parameter had " << m_setting[paramName].size() << " previous values)"); + CHECK(m_setting[paramName].size() == values.size()); + }else{ + VERBOSE(2," (the parameter does not have previous values)"); + m_setting[paramName].resize(values.size()); + } + VERBOSE(2," with the following values:"); + int i=0; + for (PARAM_VEC::iterator iter = values.begin(); iter != values.end() ; iter++, i++){ + m_setting[paramName][i] = *iter; + VERBOSE(2, " " << *iter); + } + VERBOSE(2, std::endl); +} + } diff --git a/moses/src/Parameter.h b/moses/src/Parameter.h index b521f966b..0e7b0cd24 100644 --- a/moses/src/Parameter.h +++ b/moses/src/Parameter.h @@ -42,10 +42,11 @@ typedef std::map<std::string, std::string > PARAM_STRING; class Parameter { protected: - PARAM_MAP m_setting; - PARAM_BOOL m_valid; - PARAM_STRING m_abbreviation; - PARAM_STRING m_description; + PARAM_MAP m_setting; + PARAM_BOOL m_valid; + PARAM_STRING m_abbreviation; + PARAM_STRING m_description; + PARAM_STRING m_fullname; std::string FindParam(const std::string ¶mSwitch, int argc, char* argv[]); void OverwriteParam(const std::string ¶mSwitch, const std::string ¶mName, int argc, char* argv[]); @@ -75,6 +76,31 @@ public: return m_setting.find( paramName ) != m_setting.end(); } + bool isParamShortNameSpecified(const std::string ¶mName) + { + return m_setting.find( GetFullName(paramName) ) != m_setting.end(); + } + + const std::string GetFullName(std::string abbr) + { + return m_fullname[abbr]; + } + + const std::string GetAbbreviation(std::string full) + { + return m_abbreviation[full]; + } + const PARAM_VEC &GetParamShortName(const std::string ¶mName) + { + return GetParam(GetFullName(paramName)); + } + + void OverwriteParam(const std::string ¶mName, PARAM_VEC values); + + void OverwriteParamShortName(const std::string ¶mShortName, PARAM_VEC values){ + OverwriteParam(GetFullName(paramShortName),values); + } + }; } diff --git a/moses/src/Phrase.cpp b/moses/src/Phrase.cpp index 45ad41d95..db808390c 100644 --- a/moses/src/Phrase.cpp +++ b/moses/src/Phrase.cpp @@ -99,6 +99,20 @@ Phrase Phrase::GetSubString(const WordsRange &wordsRange) const return retPhrase; } +Phrase Phrase::GetSubString(const WordsRange &wordsRange, FactorType factorType) const +{ + Phrase retPhrase(wordsRange.GetNumWordsCovered()); + + for (size_t currPos = wordsRange.GetStartPos() ; currPos <= wordsRange.GetEndPos() ; currPos++) + { + const Factor* f = GetFactor(currPos, factorType); + Word &word = retPhrase.AddWord(); + word.SetFactor(factorType, f); + } + + return retPhrase; +} + std::string Phrase::GetStringRep(const vector<FactorType> factorsToPrint) const { stringstream strme; diff --git a/moses/src/Phrase.h b/moses/src/Phrase.h index 547f6cbcd..49586271a 100644 --- a/moses/src/Phrase.h +++ b/moses/src/Phrase.h @@ -27,6 +27,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include <vector> #include <list> #include <string> + +#include <boost/functional/hash.hpp> + #include "Word.h" #include "WordsBitmap.h" #include "TypeDef.h" @@ -124,44 +127,55 @@ public: void AddWord(const Word &newWord) { AddWord() = newWord; } - - /** appends a phrase at the end of current phrase **/ - void Append(const Phrase &endPhrase); - void PrependWord(const Word &newWord); - - void Clear() { - m_words.clear(); - } - - void RemoveWord(size_t pos) { - CHECK(pos < m_words.size()); - m_words.erase(m_words.begin() + pos); - } - - //! create new phrase class that is a substring of this phrase - Phrase GetSubString(const WordsRange &wordsRange) const; - - //! return a string rep of the phrase. Each factor is separated by the factor delimiter as specified in StaticData class - std::string GetStringRep(const std::vector<FactorType> factorsToPrint) const; - - TO_STRING(); - - - int Compare(const Phrase &other) const; - - /** transitive comparison between 2 phrases - * used to insert & find phrase in dictionary - */ - bool operator< (const Phrase &compare) const { - return Compare(compare) < 0; - } - - bool operator== (const Phrase &compare) const { - return Compare(compare) == 0; - } - + + /** appends a phrase at the end of current phrase **/ + void Append(const Phrase &endPhrase); + void PrependWord(const Word &newWord); + + void Clear() + { + m_words.clear(); + } + + void RemoveWord(size_t pos) + { + CHECK(pos < m_words.size()); + m_words.erase(m_words.begin() + pos); + } + + //! create new phrase class that is a substring of this phrase + Phrase GetSubString(const WordsRange &wordsRange) const; + Phrase GetSubString(const WordsRange &wordsRange, FactorType factorType) const; + + //! return a string rep of the phrase. Each factor is separated by the factor delimiter as specified in StaticData class + std::string GetStringRep(const std::vector<FactorType> factorsToPrint) const; + + TO_STRING(); + + + int Compare(const Phrase &other) const; + + /** transitive comparison between 2 phrases + * used to insert & find phrase in dictionary + */ + bool operator< (const Phrase &compare) const + { + return Compare(compare) < 0; + } + + bool operator== (const Phrase &compare) const + { + return Compare(compare) == 0; + } }; +inline size_t hash_value(const Phrase& phrase) { + size_t seed = 0; + for (size_t i = 0; i < phrase.GetSize(); ++i) { + boost::hash_combine(seed, phrase.GetWord(i)); + } + return seed; +} } #endif diff --git a/moses/src/PhraseBoundaryFeature.cpp b/moses/src/PhraseBoundaryFeature.cpp new file mode 100644 index 000000000..461fcbe57 --- /dev/null +++ b/moses/src/PhraseBoundaryFeature.cpp @@ -0,0 +1,96 @@ +#include "PhraseBoundaryFeature.h" + +#include "Hypothesis.h" + +using namespace std; + +namespace Moses { + +int PhraseBoundaryState::Compare(const FFState& other) const +{ + const PhraseBoundaryState& rhs = dynamic_cast<const PhraseBoundaryState&>(other); + int tgt = Word::Compare(*m_targetWord,*(rhs.m_targetWord)); + if (tgt) return tgt; + return Word::Compare(*m_sourceWord,*(rhs.m_sourceWord)); +} + + +PhraseBoundaryFeature::PhraseBoundaryFeature + (const FactorList& sourceFactors, const FactorList& targetFactors) : + StatefulFeatureFunction("pb", ScoreProducer::unlimited), m_sourceFactors(sourceFactors), + m_targetFactors(targetFactors), m_sparseProducerWeight(1) +{ +} + +string PhraseBoundaryFeature::GetScoreProducerWeightShortName(unsigned) const +{ + return "pb"; +} + +size_t PhraseBoundaryFeature::GetNumInputScores() const +{ + return 0; +} + +const FFState* PhraseBoundaryFeature::EmptyHypothesisState(const InputType &) const +{ + return new PhraseBoundaryState(NULL,NULL); +} + + +void PhraseBoundaryFeature::AddFeatures( + const Word* leftWord, const Word* rightWord, const FactorList& factors, const string& side, + ScoreComponentCollection* scores) const { + for (size_t i = 0; i < factors.size(); ++i) { + ostringstream name; + name << side << ":"; + name << factors[i]; + name << ":"; + if (leftWord) { + name << leftWord->GetFactor(factors[i])->GetString(); + } else { + name << BOS_; + } + name << ":"; + if (rightWord) { + name << rightWord->GetFactor(factors[i])->GetString(); + } else { + name << EOS_; + } + scores->PlusEquals(this,name.str(),1); + } + +} + +FFState* PhraseBoundaryFeature::Evaluate + (const Hypothesis& cur_hypo, const FFState* prev_state, + ScoreComponentCollection* scores) const +{ + const PhraseBoundaryState* pbState = dynamic_cast<const PhraseBoundaryState*>(prev_state); + const Phrase& targetPhrase = cur_hypo.GetCurrTargetPhrase(); + if (targetPhrase.GetSize() == 0) { + return new PhraseBoundaryState(*pbState); + } + const Word* leftTargetWord = pbState->GetTargetWord(); + const Word* rightTargetWord = &(targetPhrase.GetWord(0)); + AddFeatures(leftTargetWord,rightTargetWord,m_targetFactors,"tgt",scores); + + const Phrase* sourcePhrase = cur_hypo.GetSourcePhrase(); + const Word* leftSourceWord = pbState->GetSourceWord(); + const Word* rightSourceWord = &(sourcePhrase->GetWord(0)); + AddFeatures(leftSourceWord,rightSourceWord,m_sourceFactors,"src",scores); + + const Word* endSourceWord = &(sourcePhrase->GetWord(sourcePhrase->GetSize()-1)); + const Word* endTargetWord = &(targetPhrase.GetWord(targetPhrase.GetSize()-1)); + + //if end of sentence add EOS + if (cur_hypo.IsSourceCompleted()) { + AddFeatures(endSourceWord,NULL,m_sourceFactors,"src",scores); + AddFeatures(endTargetWord,NULL,m_targetFactors,"tgt",scores); + } + + return new PhraseBoundaryState(endSourceWord,endTargetWord); +} + + +} diff --git a/moses/src/PhraseBoundaryFeature.h b/moses/src/PhraseBoundaryFeature.h new file mode 100644 index 000000000..4167ab9b2 --- /dev/null +++ b/moses/src/PhraseBoundaryFeature.h @@ -0,0 +1,67 @@ +#ifndef moses_PhraseBoundaryFeature_h +#define moses_PhraseBoundaryFeature_h + +#include <sstream> +#include <string> + +#include "FeatureFunction.h" +#include "FFState.h" +#include "Word.h" + +namespace Moses +{ + +class PhraseBoundaryState : public FFState { +public: + PhraseBoundaryState(const Word* sourceWord, const Word* targetWord) : + m_sourceWord(sourceWord), m_targetWord(targetWord) {} + const Word* GetSourceWord() const {return m_sourceWord;} + const Word* GetTargetWord() const {return m_targetWord;} + virtual int Compare(const FFState& other) const; + + +private: + const Word* m_sourceWord; + const Word* m_targetWord; +}; + + +/** + * Concatenations of factors on boundaries of phrases. + **/ +class PhraseBoundaryFeature : public StatefulFeatureFunction { +public: + PhraseBoundaryFeature(const FactorList& sourceFactors, const FactorList& targetFactors); + + size_t GetNumScoreComponents() const; + std::string GetScoreProducerWeightShortName(unsigned) const; + size_t GetNumInputScores() const; + + virtual const FFState* EmptyHypothesisState(const InputType &) const; + + virtual FFState* Evaluate(const Hypothesis& cur_hypo, const FFState* prev_state, + ScoreComponentCollection* accumulator) const; + + virtual FFState* EvaluateChart( const ChartHypothesis& /* cur_hypo */, + int /* featureID */, + ScoreComponentCollection* ) const + { + abort(); + } + + void SetSparseProducerWeight(float weight) { m_sparseProducerWeight = weight; } + float GetSparseProducerWeight() const { return m_sparseProducerWeight; } + +private: + void AddFeatures( + const Word* leftWord, const Word* rightWord, const FactorList& factors, + const std::string& side, ScoreComponentCollection* scores) const ; + FactorList m_sourceFactors; + FactorList m_targetFactors; + float m_sparseProducerWeight; +}; + +} + + +#endif diff --git a/moses/src/PhraseDictionary.cpp b/moses/src/PhraseDictionary.cpp index 5b353ddd5..d1992a9f4 100644 --- a/moses/src/PhraseDictionary.cpp +++ b/moses/src/PhraseDictionary.cpp @@ -48,32 +48,41 @@ GetTargetPhraseCollection(InputType const& src,WordsRange const& range) const return GetTargetPhraseCollection(src.GetSubString(range)); } +size_t PhraseDictionary::GetDictIndex() const +{ + return m_feature->GetDictIndex(); +} + PhraseDictionaryFeature::PhraseDictionaryFeature (PhraseTableImplementation implementation + , SparsePhraseDictionaryFeature* spdf , size_t numScoreComponent , unsigned numInputScores , const std::vector<FactorType> &input , const std::vector<FactorType> &output , const std::string &filePath , const std::vector<float> &weight + , size_t dictIndex , size_t tableLimit , const std::string &targetFile // default param , const std::string &alignmentsFile) // default param - :DecodeFeature(input,output) - , m_numScoreComponent(numScoreComponent), + :DecodeFeature("PhraseModel",numScoreComponent,input,output), + m_dictIndex(dictIndex), m_numInputScores(numInputScores), m_filePath(filePath), - m_weight(weight), m_tableLimit(tableLimit), m_implementation(implementation), m_targetFile(targetFile), - m_alignmentsFile(alignmentsFile) + m_alignmentsFile(alignmentsFile), + m_sparsePhraseDictionaryFeature(spdf) { - const StaticData& staticData = StaticData::Instance(); - const_cast<ScoreIndexManager&>(staticData.GetScoreIndexManager()).AddScoreProducer(this); - if (implementation == Memory || implementation == SCFG || implementation == SuffixArray - || implementation == Compact) { + if (implementation == Memory || implementation == SCFG || implementation == SuffixArray || + implementation==Compact) { m_useThreadSafePhraseDictionary = true; + if (implementation == SuffixArray) { + cerr << "Warning: implementation holds chached weights!" << endl; + exit(1); + } } else { m_useThreadSafePhraseDictionary = false; } @@ -82,6 +91,8 @@ PhraseDictionaryFeature::PhraseDictionaryFeature PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSystem* system) { const StaticData& staticData = StaticData::Instance(); + std::vector<float> weightT = staticData.GetWeights(this); + if (m_implementation == Memory) { // memory phrase table VERBOSE(2,"using standard phrase tables" << std::endl); @@ -94,21 +105,21 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst CHECK(false); } - PhraseDictionaryMemory* pdm = new PhraseDictionaryMemory(m_numScoreComponent,this); + PhraseDictionaryMemory* pdm = new PhraseDictionaryMemory(GetNumScoreComponents(),this); bool ret = pdm->Load(GetInput(), GetOutput() , m_filePath - , m_weight + , weightT , m_tableLimit , system->GetLanguageModels() , system->GetWeightWordPenalty()); CHECK(ret); return pdm; } else if (m_implementation == Binary) { - PhraseDictionaryTreeAdaptor* pdta = new PhraseDictionaryTreeAdaptor(m_numScoreComponent, m_numInputScores,this); + PhraseDictionaryTreeAdaptor* pdta = new PhraseDictionaryTreeAdaptor(GetNumScoreComponents(), m_numInputScores,this); bool ret = pdta->Load( GetInput() , GetOutput() , m_filePath - , m_weight + , weightT , m_tableLimit , system->GetLanguageModels() , system->GetWeightWordPenalty()); @@ -128,32 +139,34 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst RuleTableTrie *dict; if (staticData.GetParsingAlgorithm() == ParseScope3) { - dict = new RuleTableUTrie(m_numScoreComponent, this); + dict = new RuleTableUTrie(GetNumScoreComponents(), this); } else { - dict = new PhraseDictionarySCFG(m_numScoreComponent, this); + dict = new PhraseDictionarySCFG(GetNumScoreComponents(), this); } bool ret = dict->Load(GetInput() , GetOutput() , m_filePath - , m_weight + , weightT , m_tableLimit , system->GetLanguageModels() , system->GetWordPenaltyProducer()); - assert(ret); + CHECK(ret); return dict; } else if (m_implementation == ALSuffixArray) { // memory phrase table + cerr << "Warning: Implementation holds cached weights!" << endl; + exit(1); VERBOSE(2,"using Hiero format phrase tables" << std::endl); if (!FileExists(m_filePath) && FileExists(m_filePath + ".gz")) { m_filePath += ".gz"; VERBOSE(2,"Using gzipped file" << std::endl); } - PhraseDictionaryALSuffixArray* pdm = new PhraseDictionaryALSuffixArray(m_numScoreComponent,this); + PhraseDictionaryALSuffixArray* pdm = new PhraseDictionaryALSuffixArray(GetNumScoreComponents(),this); bool ret = pdm->Load(GetInput() , GetOutput() , m_filePath - , m_weight + , weightT , m_tableLimit , system->GetLanguageModels() , system->GetWordPenaltyProducer()); @@ -161,28 +174,30 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst return pdm; } else if (m_implementation == OnDisk) { - PhraseDictionaryOnDisk* pdta = new PhraseDictionaryOnDisk(m_numScoreComponent, this); + PhraseDictionaryOnDisk* pdta = new PhraseDictionaryOnDisk(GetNumScoreComponents(), this); bool ret = pdta->Load(GetInput() , GetOutput() , m_filePath - , m_weight + , weightT , m_tableLimit , system->GetLanguageModels() , system->GetWordPenaltyProducer()); CHECK(ret); return pdta; } else if (m_implementation == SuffixArray) { + cerr << "Warning: Implementation holds cached weights!" << endl; + exit(1); #ifndef WIN32 - PhraseDictionaryDynSuffixArray *pd = new PhraseDictionaryDynSuffixArray(m_numScoreComponent, this); + PhraseDictionaryDynSuffixArray *pd = new PhraseDictionaryDynSuffixArray(GetNumScoreComponents(), this); if(!(pd->Load( GetInput() ,GetOutput() ,m_filePath ,m_targetFile - , m_alignmentsFile - , m_weight, m_tableLimit - , system->GetLanguageModels() - , system->GetWeightWordPenalty()))) { + ,m_alignmentsFile + ,weightT, m_tableLimit + ,system->GetLanguageModels() + ,system->GetWeightWordPenalty()))) { std::cerr << "FAILED TO LOAD\n" << endl; delete pd; pd = NULL; @@ -194,30 +209,30 @@ PhraseDictionary* PhraseDictionaryFeature::LoadPhraseTable(const TranslationSyst #endif } else if (m_implementation == FuzzyMatch) { - PhraseDictionaryFuzzyMatch *dict = new PhraseDictionaryFuzzyMatch(m_numScoreComponent, this); + PhraseDictionaryFuzzyMatch *dict = new PhraseDictionaryFuzzyMatch(GetNumScoreComponents(), this); bool ret = dict->Load(GetInput() , GetOutput() , m_filePath - , m_weight + , weightT , m_tableLimit , system->GetLanguageModels() , system->GetWordPenaltyProducer()); - assert(ret); + CHECK(ret); return dict; } else if (m_implementation == Compact) { #ifndef WIN32 VERBOSE(2,"Using compact phrase table" << std::endl); - PhraseDictionaryCompact* pd = new PhraseDictionaryCompact(m_numScoreComponent, m_implementation, this); + PhraseDictionaryCompact* pd = new PhraseDictionaryCompact(GetNumScoreComponents(), m_implementation, this); bool ret = pd->Load(GetInput(), GetOutput() , m_filePath - , m_weight + , weightT , m_tableLimit , system->GetLanguageModels() , system->GetWeightWordPenalty()); - assert(ret); + CHECK(ret); return pd; #else CHECK(false); @@ -272,21 +287,23 @@ const PhraseDictionary* PhraseDictionaryFeature::GetDictionary() const return dict; } +PhraseDictionary* PhraseDictionaryFeature::GetDictionary() +{ + PhraseDictionary* dict; + if (m_useThreadSafePhraseDictionary) { + dict = m_threadSafePhraseDictionary.get(); + } else { + dict = m_threadUnsafePhraseDictionary.get(); + } + CHECK(dict); + return dict; +} PhraseDictionaryFeature::~PhraseDictionaryFeature() {} -std::string PhraseDictionaryFeature::GetScoreProducerDescription(unsigned idx) const -{ - if (idx < GetNumInputScores()){ - return "InputScore"; - }else{ - return "PhraseModel"; - } -} - std::string PhraseDictionaryFeature::GetScoreProducerWeightShortName(unsigned idx) const { if (idx < GetNumInputScores()){ @@ -296,10 +313,6 @@ std::string PhraseDictionaryFeature::GetScoreProducerWeightShortName(unsigned id } } -size_t PhraseDictionaryFeature::GetNumScoreComponents() const -{ - return m_numScoreComponent; -} size_t PhraseDictionaryFeature::GetNumInputScores() const { @@ -316,5 +329,10 @@ const PhraseDictionaryFeature* PhraseDictionary::GetFeature() const return m_feature; } +size_t PhraseDictionaryFeature::GetDictIndex() const +{ + return m_dictIndex; +} + } diff --git a/moses/src/PhraseDictionary.h b/moses/src/PhraseDictionary.h index 7ec4162a0..3136c47de 100644 --- a/moses/src/PhraseDictionary.h +++ b/moses/src/PhraseDictionary.h @@ -26,6 +26,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include <map> #include <memory> #include <list> +#include <stdexcept> #include <vector> #include <string> @@ -51,6 +52,7 @@ class TranslationSystem; class ChartRuleLookupManager; class PhraseDictionaryFeature; +class SparsePhraseDictionaryFeature; /** * Abstract base class for phrase dictionaries (tables). @@ -68,6 +70,7 @@ public: return Translate; } const PhraseDictionaryFeature* GetFeature() const; + size_t GetDictIndex() const; //! find list of translations that can translates src. Only for phrase input virtual const TargetPhraseCollection *GetTargetPhraseCollection(const Phrase& src) const=0; @@ -96,12 +99,14 @@ class PhraseDictionaryFeature : public DecodeFeature public: PhraseDictionaryFeature( PhraseTableImplementation implementation + , SparsePhraseDictionaryFeature* spdf , size_t numScoreComponent , unsigned numInputScores , const std::vector<FactorType> &input , const std::vector<FactorType> &output , const std::string &filePath , const std::vector<float> &weight + , size_t dictIndex , size_t tableLimit , const std::string &targetFile , const std::string &alignmentsFile); @@ -111,13 +116,14 @@ public: virtual bool ComputeValueInTranslationOption() const; - std::string GetScoreProducerDescription(unsigned) const; std::string GetScoreProducerWeightShortName(unsigned idx=0) const; - size_t GetNumScoreComponents() const; - size_t GetNumInputScores() const; + SparsePhraseDictionaryFeature* GetSparsePhraseDictionaryFeature() const { + return m_sparsePhraseDictionaryFeature; + } + //Initialises the dictionary (may involve loading from file) void InitDictionary(const TranslationSystem* system); @@ -126,15 +132,34 @@ public: //Get the dictionary. Be sure to initialise it first. const PhraseDictionary* GetDictionary() const; + PhraseDictionary* GetDictionary(); + size_t GetDictIndex() const; + + //Usual feature function methods are not implemented + virtual void Evaluate(const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const + { + throw std::logic_error("PhraseDictionary.Evaluate() Not implemented"); + } + + virtual void EvaluateChart(const ChartBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const + { + throw std::logic_error("PhraseDictionary.EvaluateChart() Not implemented"); + } + + virtual bool ComputeValueInTranslationTable() const {return true;} + + +protected: + size_t m_dictIndex; private: /** Load the appropriate phrase table */ PhraseDictionary* LoadPhraseTable(const TranslationSystem* system); - size_t m_numScoreComponent; unsigned m_numInputScores; std::string m_filePath; - std::vector<float> m_weight; size_t m_tableLimit; //We instantiate either the the thread-safe or non-thread-safe dictionary, //but not both. The thread-safe one can be instantiated in the constructor and shared @@ -151,6 +176,7 @@ private: PhraseTableImplementation m_implementation; std::string m_targetFile; std::string m_alignmentsFile; + SparsePhraseDictionaryFeature* m_sparsePhraseDictionaryFeature; }; diff --git a/moses/src/PhraseDictionaryDynSuffixArray.cpp b/moses/src/PhraseDictionaryDynSuffixArray.cpp index 5d65c00c6..8e32758fa 100644 --- a/moses/src/PhraseDictionaryDynSuffixArray.cpp +++ b/moses/src/PhraseDictionaryDynSuffixArray.cpp @@ -61,7 +61,7 @@ const TargetPhraseCollection *PhraseDictionaryDynSuffixArray::GetTargetPhraseCol TargetPhrase *targetPhrase = itr->second; //std::transform(scoreVector.begin(),scoreVector.end(),scoreVector.begin(),NegateScore); std::transform(scoreVector.begin(),scoreVector.end(),scoreVector.begin(),FloorScore); - targetPhrase->SetScore(m_feature, scoreVector, m_weight, m_weightWP, *m_languageModels); + targetPhrase->SetScore(m_feature, scoreVector, ScoreComponentCollection(), m_weight, m_weightWP, *m_languageModels); //cout << *targetPhrase << "\t" << std::setprecision(8) << scoreVector[2] << endl; ret->Add(targetPhrase); } diff --git a/moses/src/PhraseDictionaryMemory.cpp b/moses/src/PhraseDictionaryMemory.cpp index 3915d146c..9eaaa6dec 100644 --- a/moses/src/PhraseDictionaryMemory.cpp +++ b/moses/src/PhraseDictionaryMemory.cpp @@ -38,6 +38,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "StaticData.h" #include "WordsRange.h" #include "UserMessage.h" +#include "SparsePhraseDictionaryFeature.h" using namespace std; @@ -106,7 +107,6 @@ bool PhraseDictionaryMemory::Load(const std::vector<FactorType> &input //target std::auto_ptr<TargetPhrase> targetPhrase(new TargetPhrase(Output)); - targetPhrase->SetSourcePhrase(&sourcePhrase); // TODO(bhaddow): This is a dangling pointer targetPhrase->CreateFromString(output, targetPhraseString, factorDelimiter); scv.clear(); @@ -127,14 +127,30 @@ bool PhraseDictionaryMemory::Load(const std::vector<FactorType> &input UserMessage::Add(strme.str()); abort(); } - // scv good to go sir! - targetPhrase->SetScore(m_feature, scv, weight, weightWP, languageModels); + + size_t consumed = 3; if (pipes) { targetPhrase->SetAlignmentInfo(*pipes++); ++consumed; } + + ScoreComponentCollection sparse; + if (pipes) pipes++; //counts + if (pipes) { + //sparse features + SparsePhraseDictionaryFeature* spdf = + GetFeature()->GetSparsePhraseDictionaryFeature(); + if (spdf) { + sparse.Assign(spdf,(pipes++)->as_string()); + } + } + + + // scv good to go sir! + targetPhrase->SetScore(m_feature, scv, sparse, weight, weightWP, languageModels); + // Check number of entries delimited by ||| agrees across all lines. for (; pipes; ++pipes, ++consumed) {} if (numElement != consumed) { @@ -148,12 +164,15 @@ bool PhraseDictionaryMemory::Load(const std::vector<FactorType> &input } } - // Reuse source if possible. Otherwise, create node for it. + //TODO: Would be better to reuse source phrases, but ownership has to be + //consistent across phrase table implementations + sourcePhrase.Clear(); + sourcePhrase.CreateFromString(input, sourcePhraseString, factorDelimiter); + //Now that the source phrase is ready, we give the target phrase a copy + targetPhrase->SetSourcePhrase(sourcePhrase); if (preSourceString == sourcePhraseString && preSourceNode) { preSourceNode->Add(targetPhrase.release()); } else { - sourcePhrase.Clear(); - sourcePhrase.CreateFromString(input, sourcePhraseString, factorDelimiter); preSourceNode = CreateTargetPhraseCollection(sourcePhrase); preSourceNode->Add(targetPhrase.release()); preSourceString.assign(sourcePhraseString.data(), sourcePhraseString.size()); diff --git a/moses/src/PhraseDictionaryTree.cpp b/moses/src/PhraseDictionaryTree.cpp index c045fec0b..13b404724 100644 --- a/moses/src/PhraseDictionaryTree.cpp +++ b/moses/src/PhraseDictionaryTree.cpp @@ -1,5 +1,6 @@ // $Id$ // vim:tabstop=2 +#include "FeatureVector.h" #include "PhraseDictionaryTree.h" #include <map> #include "util/check.hh" @@ -9,6 +10,7 @@ #include <string> #include <vector> + namespace Moses { @@ -28,6 +30,11 @@ class TgtCand IPhrase e; Scores sc; std::string m_alignment; + IPhrase fnames; + std::vector<FValue> fvalues; + + static const float SPARSE_FLAG; + public: TgtCand() {} @@ -46,23 +53,37 @@ public: void writeBin(FILE* f) const { fWriteVector(f,e); - fWriteVector(f,sc); + //This is a bit ugly, but if there is a sparse vector, add + //an extra score with value 100. Can the last score be 100? + //Probably not, since scores are probabilities and phrase penalty. + if (fnames.size()) { + Scores sc_copy(sc); + sc_copy.push_back(SPARSE_FLAG); + fWriteVector(f,sc_copy); + fWriteVector(f,fnames); + fWriteVector(f,fvalues); + } else { + fWriteVector(f,sc); + } } void readBin(FILE* f) { fReadVector(f,e); fReadVector(f,sc); + if (sc.back() == 100) { + sc.pop_back(); + fReadVector(f,fnames); + fReadVector(f,fvalues); + } } void writeBinWithAlignment(FILE* f) const { - fWriteVector(f,e); - fWriteVector(f,sc); + writeBin(f); fWriteString(f, m_alignment.c_str(), m_alignment.size()); } void readBinWithAlignment(FILE* f) { - fReadVector(f,e); - fReadVector(f,sc); + readBin(f); fReadString(f, m_alignment); } @@ -75,8 +96,24 @@ public: const std::string& GetAlignment() const { return m_alignment; } + + const IPhrase& GetFeatureNames() const { + return fnames; + } + + const std::vector<FValue> GetFeatureValues() const { + return fvalues; + } + + void SetFeatures(const IPhrase& names, const std::vector<FValue>& values) { + CHECK(names.size() == values.size()); + fnames = names; + fvalues = values; + } }; +const float TgtCand::SPARSE_FLAG = 100; + class TgtCands : public std::vector<TgtCand> { @@ -214,29 +251,23 @@ public: void PrintTgtCand(const TgtCands& tcands,std::ostream& out) const; // convert target candidates from internal data structure to the external one - void ConvertTgtCand(const TgtCands& tcands,std::vector<StringTgtCand>& rv) const { - for(TgtCands::const_iterator i=tcands.begin(); i!=tcands.end(); ++i) { - const IPhrase& iphrase=i->GetPhrase(); - std::vector<std::string const*> vs; - vs.reserve(iphrase.size()); - for(size_t j=0; j<iphrase.size(); ++j) - vs.push_back(&tv->symbol(iphrase[j])); - rv.push_back(StringTgtCand(vs,i->GetScores())); - } - } - - // convert target candidates from internal data structure to the external one void ConvertTgtCand(const TgtCands& tcands,std::vector<StringTgtCand>& rv, - std::vector<std::string>& wa) const { + std::vector<std::string>* wa) const { for(TgtCands::const_iterator i=tcands.begin(); i!=tcands.end(); ++i) { + rv.push_back(StringTgtCand()); const IPhrase& iphrase=i->GetPhrase(); - std::vector<std::string const*> vs; - vs.reserve(iphrase.size()); - for(size_t j=0; j<iphrase.size(); ++j) - vs.push_back(&tv->symbol(iphrase[j])); - rv.push_back(StringTgtCand(vs,i->GetScores())); - wa.push_back(i->GetAlignment()); + rv.back().tokens.reserve(iphrase.size()); + for(size_t j=0; j<iphrase.size(); ++j) { + rv.back().tokens.push_back(&tv->symbol(iphrase[j])); + } + rv.back().scores = i->GetScores(); + const IPhrase& fnames = i->GetFeatureNames(); + for (size_t j = 0; j < fnames.size(); ++j) { + rv.back().fnames.push_back(&tv->symbol(fnames[j])); + } + rv.back().fvalues = i->GetFeatureValues(); + if (wa) wa->push_back(i->GetAlignment()); } } @@ -387,6 +418,7 @@ void PhraseDictionaryTree::FreeMemory() const imp->FreeMemory(); } + void PhraseDictionaryTree:: GetTargetCandidates(const std::vector<std::string>& src, std::vector<StringTgtCand>& rv) const @@ -399,7 +431,7 @@ GetTargetCandidates(const std::vector<std::string>& src, TgtCands tgtCands; imp->GetTargetCandidates(f,tgtCands); - imp->ConvertTgtCand(tgtCands,rv); + imp->ConvertTgtCand(tgtCands,rv,NULL); } void PhraseDictionaryTree:: @@ -415,7 +447,7 @@ GetTargetCandidates(const std::vector<std::string>& src, TgtCands tgtCands; imp->GetTargetCandidates(f,tgtCands); - imp->ConvertTgtCand(tgtCands,rv,wa); + imp->ConvertTgtCand(tgtCands,rv,&wa); } @@ -493,6 +525,7 @@ int PhraseDictionaryTree::Create(std::istream& inFile,const std::string& out) ,&scoreString = tokens[2]; const std::string empty; const std::string &alignmentString = PrintWordAlignment() ? tokens[3] : empty; + const std::string sparseFeatureString = tokens.size() > 5 ? tokens[5] : empty; IPhrase f,e; Scores sc; @@ -532,6 +565,21 @@ int PhraseDictionaryTree::Create(std::istream& inFile,const std::string& out) } } + IPhrase fnames; + std::vector<FValue> fvalues; + if (!sparseFeatureString.empty()) { + std::vector<std::string> sparseTokens = Tokenize(sparseFeatureString); + if (sparseTokens.size() % 2 != 0) { + TRACE_ERR("ERROR: incorrectly formatted sparse feature string: " << + sparseFeatureString << std::endl); + abort(); + } + for (size_t i = 0; i < sparseTokens.size(); i+=2) { + fnames.push_back(imp->tv->add(sparseTokens[i])); + fvalues.push_back(Scan<FValue>(sparseTokens[i+1])); + } + } + if(currF!=f) { // new src phrase currF=f; @@ -571,6 +619,7 @@ int PhraseDictionaryTree::Create(std::istream& inFile,const std::string& out) } tgtCands.push_back(TgtCand(e,sc, alignmentString)); CHECK(currFirstWord!=InvalidLabelId); + tgtCands.back().SetFeatures(fnames, fvalues); } if (PrintWordAlignment()) tgtCands.writeBinWithAlignment(ot); @@ -648,7 +697,7 @@ GetTargetCandidates(PrefixPtr p, { TgtCands tcands; imp->GetTargetCandidates(p,tcands); - imp->ConvertTgtCand(tcands,rv); + imp->ConvertTgtCand(tcands,rv,NULL); } void PhraseDictionaryTree:: @@ -658,7 +707,7 @@ GetTargetCandidates(PrefixPtr p, { TgtCands tcands; imp->GetTargetCandidates(p,tcands); - imp->ConvertTgtCand(tcands,rv,wa); + imp->ConvertTgtCand(tcands,rv,&wa); } std::string PhraseDictionaryTree::GetScoreProducerDescription(unsigned) const diff --git a/moses/src/PhraseDictionaryTree.h b/moses/src/PhraseDictionaryTree.h index cdf255033..b060dd8bb 100644 --- a/moses/src/PhraseDictionaryTree.h +++ b/moses/src/PhraseDictionaryTree.h @@ -33,6 +33,17 @@ class PDTimp; typedef PrefixTreeF<LabelId,OFF_T> PTF; +//typedef std::pair<std::vector<std::string const*>,Scores > StringTgtCand; +struct StringTgtCand +{ + typedef std::vector<std::string const*> Tokens; + Tokens tokens; + Scores scores; + Tokens fnames; + std::vector<FValue> fvalues; + +}; + /** A phrase table for phrase-based decoding that is held on disk, rather than in memory * Wrapper around a PDTimp class */ @@ -83,6 +94,7 @@ public: // get the target candidates for a given phrase void GetTargetCandidates(const std::vector<std::string>& src, std::vector<StringTgtCand>& rv) const; + // get the target candidates for a given phrase void GetTargetCandidates(const std::vector<std::string>& src, diff --git a/moses/src/PhraseDictionaryTreeAdaptor.cpp b/moses/src/PhraseDictionaryTreeAdaptor.cpp index cce4a2ee4..154572bce 100644 --- a/moses/src/PhraseDictionaryTreeAdaptor.cpp +++ b/moses/src/PhraseDictionaryTreeAdaptor.cpp @@ -55,8 +55,7 @@ bool PhraseDictionaryTreeAdaptor::Load(const std::vector<FactorType> &input // set PhraseDictionary members m_tableLimit=tableLimit; - imp->Create(input,output,filePath, - weight,languageModels,weightWP); + imp->Create(input,output,filePath,weight,languageModels); return true; } diff --git a/moses/src/PhraseLengthFeature.cpp b/moses/src/PhraseLengthFeature.cpp new file mode 100644 index 000000000..b7745f27c --- /dev/null +++ b/moses/src/PhraseLengthFeature.cpp @@ -0,0 +1,37 @@ +#include <sstream> +#include "PhraseLengthFeature.h" +#include "Hypothesis.h" +#include "ScoreComponentCollection.h" +#include "TranslationOption.h" + +namespace Moses { + +using namespace std; + +void PhraseLengthFeature::Evaluate( + const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const +{ + // get length of source and target phrase + size_t sourceLength = context.GetTargetPhrase().GetSize(); + size_t targetLength = context.GetTranslationOption().GetSourcePhrase()->GetSize(); + + // create feature names + stringstream nameSource; + nameSource << "s" << sourceLength; + + stringstream nameTarget; + nameTarget << "t" << targetLength; + + stringstream nameBoth; + nameBoth << sourceLength << "," << targetLength; + + // increase feature counts + accumulator->PlusEquals(this,nameSource.str(),1); + accumulator->PlusEquals(this,nameTarget.str(),1); + accumulator->PlusEquals(this,nameBoth.str(),1); + + //cerr << nameSource.str() << " " << nameTarget.str() << " " << nameBoth.str() << endl; +} + +} diff --git a/moses/src/PhraseLengthFeature.h b/moses/src/PhraseLengthFeature.h new file mode 100644 index 000000000..041db0f0e --- /dev/null +++ b/moses/src/PhraseLengthFeature.h @@ -0,0 +1,39 @@ +#ifndef moses_PhraseLengthFeature_h +#define moses_PhraseLengthFeature_h + +#include <stdexcept> +#include <string> +#include <map> + +#include "FactorCollection.h" +#include "FeatureFunction.h" +#include "FFState.h" +#include "Word.h" + +namespace Moses +{ + +/** Sets the features for length of source phrase, target phrase, both. + */ +class PhraseLengthFeature : public StatelessFeatureFunction { +public: + PhraseLengthFeature(): + StatelessFeatureFunction("pl", ScoreProducer::unlimited) + {} + + void Evaluate(const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const; + + void EvaluateChart(const ChartBasedFeatureContext& context, + ScoreComponentCollection*) const { + throw std::logic_error("PhraseLengthFeature not valid in chart decoder"); + } + + // basic properties + std::string GetScoreProducerWeightShortName(unsigned) const { return "pl"; } + size_t GetNumInputScores() const { return 0; } +}; + +} + +#endif // moses_PhraseLengthFeature_h diff --git a/moses/src/PhrasePairFeature.cpp b/moses/src/PhrasePairFeature.cpp new file mode 100644 index 000000000..020292748 --- /dev/null +++ b/moses/src/PhrasePairFeature.cpp @@ -0,0 +1,226 @@ +#include <boost/algorithm/string.hpp> + +#include "AlignmentInfo.h" +#include "PhrasePairFeature.h" +#include "TargetPhrase.h" +#include "Hypothesis.h" +#include "TranslationOption.h" +#include <boost/algorithm/string.hpp> + +using namespace std; + +namespace Moses { + +string PhrasePairFeature::GetScoreProducerWeightShortName(unsigned) const +{ + return "pp"; +} + +size_t PhrasePairFeature::GetNumInputScores() const +{ + return 0; +} + +bool PhrasePairFeature::Load(const std::string &filePathSource/*, const std::string &filePathTarget*/) +{ + if (m_domainTrigger) { + // domain trigger terms for each input document + ifstream inFileSource(filePathSource.c_str()); + if (!inFileSource) + { + cerr << "could not open file " << filePathSource << endl; + return false; + } + + std::string line; + while (getline(inFileSource, line)) { + std::set<std::string> terms; + vector<string> termVector; + boost::split(termVector, line, boost::is_any_of("\t ")); + for (size_t i=0; i < termVector.size(); ++i) + terms.insert(termVector[i]); + + // add term set for current document + m_vocabDomain.push_back(terms); + } + + inFileSource.close(); + } + else { + // restricted source word vocabulary + ifstream inFileSource(filePathSource.c_str()); + if (!inFileSource) + { + cerr << "could not open file " << filePathSource << endl; + return false; + } + + std::string line; + while (getline(inFileSource, line)) { + m_vocabSource.insert(line); + } + + inFileSource.close(); + + /* // restricted target word vocabulary + ifstream inFileTarget(filePathTarget.c_str()); + if (!inFileTarget) + { + cerr << "could not open file " << filePathTarget << endl; + return false; + } + + while (getline(inFileTarget, line)) { + m_vocabTarget.insert(line); + } + + inFileTarget.close();*/ + + m_unrestricted = false; + } + return true; +} + +void PhrasePairFeature::Evaluate( + const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const +{ + const TargetPhrase& target = context.GetTargetPhrase(); + const Phrase& source = *(context.GetTranslationOption().GetSourcePhrase()); + if (m_simple) { + ostringstream namestr; + namestr << "pp_"; + namestr << source.GetWord(0).GetFactor(m_sourceFactorId)->GetString(); + for (size_t i = 1; i < source.GetSize(); ++i) { + const Factor* sourceFactor = source.GetWord(i).GetFactor(m_sourceFactorId); + namestr << ","; + namestr << sourceFactor->GetString(); + } + namestr << "~"; + namestr << target.GetWord(0).GetFactor(m_targetFactorId)->GetString(); + for (size_t i = 1; i < target.GetSize(); ++i) { + const Factor* targetFactor = target.GetWord(i).GetFactor(m_targetFactorId); + namestr << ","; + namestr << targetFactor->GetString(); + } + + accumulator->SparsePlusEquals(namestr.str(),1); + } + if (m_domainTrigger) { + const Sentence& input = static_cast<const Sentence&>(context.GetSource()); + const bool use_topicid = input.GetUseTopicId(); + const bool use_topicid_prob = input.GetUseTopicIdAndProb(); + + // compute pair + ostringstream pair; + pair << source.GetWord(0).GetFactor(m_sourceFactorId)->GetString(); + for (size_t i = 1; i < source.GetSize(); ++i) { + const Factor* sourceFactor = source.GetWord(i).GetFactor(m_sourceFactorId); + pair << ","; + pair << sourceFactor->GetString(); + } + pair << "~"; + pair << target.GetWord(0).GetFactor(m_targetFactorId)->GetString(); + for (size_t i = 1; i < target.GetSize(); ++i) { + const Factor* targetFactor = target.GetWord(i).GetFactor(m_targetFactorId); + pair << ","; + pair << targetFactor->GetString(); + } + + if (use_topicid || use_topicid_prob) { + if(use_topicid) { + // use topicid as trigger + const long topicid = input.GetTopicId(); + stringstream feature; + feature << "pp_"; + if (topicid == -1) + feature << "unk"; + else + feature << topicid; + + feature << "_"; + feature << pair.str(); + accumulator->SparsePlusEquals(feature.str(), 1); + } + else { + // use topic probabilities + const vector<string> &topicid_prob = *(input.GetTopicIdAndProb()); + if (atol(topicid_prob[0].c_str()) == -1) { + stringstream feature; + feature << "pp_unk_"; + feature << pair.str(); + accumulator->SparsePlusEquals(feature.str(), 1); + } + else { + for (size_t i=0; i+1 < topicid_prob.size(); i+=2) { + stringstream feature; + feature << "pp_"; + feature << topicid_prob[i]; + feature << "_"; + feature << pair.str(); + accumulator->SparsePlusEquals(feature.str(), atof((topicid_prob[i+1]).c_str())); + } + } + } + } + else { + // range over domain trigger words + const long docid = input.GetDocumentId(); + for (set<string>::const_iterator p = m_vocabDomain[docid].begin(); p != m_vocabDomain[docid].end(); ++p) { + string sourceTrigger = *p; + ostringstream namestr; + namestr << "pp_"; + namestr << sourceTrigger; + namestr << "_"; + namestr << pair.str(); + accumulator->SparsePlusEquals(namestr.str(),1); + } + } + } + if (m_sourceContext) { + const Sentence& input = static_cast<const Sentence&>(context.GetSource()); + + // range over source words to get context + for(size_t contextIndex = 0; contextIndex < input.GetSize(); contextIndex++ ) { + string sourceTrigger = input.GetWord(contextIndex).GetFactor(m_sourceFactorId)->GetString(); + if (m_ignorePunctuation) { + // check if trigger is punctuation + char firstChar = sourceTrigger.at(0); + CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar ); + if(charIterator != m_punctuationHash.end()) + continue; + } + + bool sourceTriggerExists = false; + if (!m_unrestricted) + sourceTriggerExists = m_vocabSource.find( sourceTrigger ) != m_vocabSource.end(); + + if (m_unrestricted || sourceTriggerExists) { + ostringstream namestr; + namestr << "pp_"; + namestr << sourceTrigger; + namestr << "~"; + namestr << source.GetWord(0).GetFactor(m_sourceFactorId)->GetString(); + for (size_t i = 1; i < source.GetSize(); ++i) { + const Factor* sourceFactor = source.GetWord(i).GetFactor(m_sourceFactorId); + namestr << ","; + namestr << sourceFactor->GetString(); + } + namestr << "~"; + namestr << target.GetWord(0).GetFactor(m_targetFactorId)->GetString(); + for (size_t i = 1; i < target.GetSize(); ++i) { + const Factor* targetFactor = target.GetWord(i).GetFactor(m_targetFactorId); + namestr << ","; + namestr << targetFactor->GetString(); + } + + accumulator->SparsePlusEquals(namestr.str(),1); + } + } + } +} + +bool PhrasePairFeature::ComputeValueInTranslationOption() const { + return true; +} +} diff --git a/moses/src/PhrasePairFeature.h b/moses/src/PhrasePairFeature.h new file mode 100644 index 000000000..d7cc3ea48 --- /dev/null +++ b/moses/src/PhrasePairFeature.h @@ -0,0 +1,81 @@ +#ifndef moses_PhrasePairFeature_h +#define moses_PhrasePairFeature_h + +#include <stdexcept> + +#include "Factor.h" +#include "FeatureFunction.h" +#include "Sentence.h" + +namespace Moses { + +/** + * Phrase pair feature: complete source/target phrase pair + **/ +class PhrasePairFeature: public StatelessFeatureFunction { + + typedef std::map< char, short > CharHash; + typedef std::vector< std::set<std::string> > DocumentVector; + + std::set<std::string> m_vocabSource; + //std::set<std::string> m_vocabTarget; + DocumentVector m_vocabDomain; + FactorType m_sourceFactorId; + FactorType m_targetFactorId; + bool m_unrestricted; + bool m_simple; + bool m_sourceContext; + bool m_domainTrigger; + float m_sparseProducerWeight; + bool m_ignorePunctuation; + CharHash m_punctuationHash; + + public: + PhrasePairFeature (FactorType sourceFactorId, FactorType targetFactorId, + bool simple, bool sourceContext, bool ignorePunctuation, bool domainTrigger) : + StatelessFeatureFunction("pp", ScoreProducer::unlimited), + m_sourceFactorId(sourceFactorId), + m_targetFactorId(targetFactorId), + m_unrestricted(true), + m_simple(simple), + m_sourceContext(sourceContext), + m_domainTrigger(domainTrigger), + m_sparseProducerWeight(1), + m_ignorePunctuation(ignorePunctuation) { + std::cerr << "Creating phrase pair feature.. " << std::endl; + if (m_simple == 1) std::cerr << "using simple phrase pairs.. "; + if (m_sourceContext == 1) std::cerr << "using source context.. "; + if (m_domainTrigger == 1) std::cerr << "using domain triggers.. "; + + // compile a list of punctuation characters + if (m_ignorePunctuation) { + std::cerr << "ignoring punctuation for triggers.. "; + char punctuation[] = "\"'!?¿·()#_,.:;•&@‑/\\0123456789~="; + for (size_t i=0; i < sizeof(punctuation)-1; ++i) + m_punctuationHash[punctuation[i]] = 1; + } + } + + void Evaluate(const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const; + + void EvaluateChart(const ChartBasedFeatureContext& context, + ScoreComponentCollection*) const { + throw std::logic_error("PhrasePairFeature not valid in chart decoder"); + } + + bool ComputeValueInTranslationOption() const; + + std::string GetScoreProducerWeightShortName(unsigned) const; + size_t GetNumInputScores() const; + + bool Load(const std::string &filePathSource/*, const std::string &filePathTarget*/); + + void SetSparseProducerWeight(float weight) { m_sparseProducerWeight = weight; } + float GetSparseProducerWeight() const { return m_sparseProducerWeight; } +}; + +} + + +#endif diff --git a/moses/src/RuleTable/LoaderCompact.cpp b/moses/src/RuleTable/LoaderCompact.cpp index f88c425e1..f8d8baa13 100644 --- a/moses/src/RuleTable/LoaderCompact.cpp +++ b/moses/src/RuleTable/LoaderCompact.cpp @@ -72,7 +72,7 @@ bool RuleTableLoaderCompact::Load(const std::vector<FactorType> &input, // Load alignments. std::vector<const AlignmentInfo *> alignmentSets; - LoadAlignmentSection(reader, alignmentSets); + LoadAlignmentSection(reader, alignmentSets, sourcePhrases); // Load rules. if (!LoadRuleSection(reader, vocab, sourcePhrases, targetPhrases, @@ -136,7 +136,7 @@ void RuleTableLoaderCompact::LoadPhraseSection( } void RuleTableLoaderCompact::LoadAlignmentSection( - LineReader &reader, std::vector<const AlignmentInfo *> &alignmentSets) + LineReader &reader, std::vector<const AlignmentInfo *> &alignmentSets, std::vector<Phrase> &sourcePhrases) { // Read alignment set count. reader.ReadLine(); @@ -153,13 +153,16 @@ void RuleTableLoaderCompact::LoadAlignmentSection( reader.ReadLine(); Tokenize(tokens, reader.m_line); std::vector<std::string>::const_iterator p; + int indicator[tokens.size()]; + size_t index = 0; for (p = tokens.begin(); p != tokens.end(); ++p) { points.clear(); Tokenize<size_t>(points, *p, "-"); std::pair<size_t, size_t> alignmentPair(points[0], points[1]); alignmentInfo.insert(alignmentPair); + indicator[index++] = sourcePhrases[i].GetWord(points[0]).IsNonTerminal() ? 1: 0; } - alignmentSets[i] = AlignmentInfoCollection::Instance().Add(alignmentInfo); + alignmentSets[i] = AlignmentInfoCollection::Instance().Add(alignmentInfo, indicator); } } @@ -226,6 +229,7 @@ bool RuleTableLoaderCompact::LoadRuleSection( targetPhrase->SetTargetLHS(targetLhs); targetPhrase->SetScoreChart(ruleTable.GetFeature(), scoreVector, weights, languageModels, wpProducer); + targetPhrase->SetSourcePhrase(sourcePhrase); // Insert rule into table. TargetPhraseCollection &coll = GetOrCreateTargetPhraseCollection( diff --git a/moses/src/RuleTable/LoaderCompact.h b/moses/src/RuleTable/LoaderCompact.h index 4dc7c87aa..42453ac1c 100644 --- a/moses/src/RuleTable/LoaderCompact.h +++ b/moses/src/RuleTable/LoaderCompact.h @@ -71,7 +71,8 @@ class RuleTableLoaderCompact : public RuleTableLoader std::vector<size_t> &); void LoadAlignmentSection(LineReader &, - std::vector<const AlignmentInfo *> &); + std::vector<const AlignmentInfo *> &, + std::vector<Phrase> &); bool LoadRuleSection(LineReader &, const std::vector<Word> &, diff --git a/moses/src/RuleTable/LoaderStandard.cpp b/moses/src/RuleTable/LoaderStandard.cpp index 331a07de0..5aa5df255 100644 --- a/moses/src/RuleTable/LoaderStandard.cpp +++ b/moses/src/RuleTable/LoaderStandard.cpp @@ -181,10 +181,15 @@ bool RuleTableLoaderStandard::Load(FormatType format abort(); } + if (tokens.size() == 4) { + tokens.push_back("1 1"); //dummy rule count for glue rules + } + const string &sourcePhraseString = tokens[0] , &targetPhraseString = tokens[1] , &scoreString = tokens[2] - , &alignString = tokens[3]; + , &alignString = tokens[3] + , &ruleCountString = tokens[4]; bool isLHSEmpty = (sourcePhraseString.find_first_not_of(" \t", 0) == string::npos); if (isLHSEmpty && !staticData.IsWordDeletionEnabled()) { @@ -215,12 +220,17 @@ bool RuleTableLoaderStandard::Load(FormatType format // create target phrase obj TargetPhrase *targetPhrase = new TargetPhrase(Output); targetPhrase->CreateFromStringNewFormat(Output, output, targetPhraseString, factorDelimiter, targetLHS); + targetPhrase->SetSourcePhrase(sourcePhrase); // rest of target phrase - targetPhrase->SetAlignmentInfo(alignString); + targetPhrase->SetAlignmentInfo(alignString, sourcePhrase); targetPhrase->SetTargetLHS(targetLHS); + + if (tokens.size() == 5) { + targetPhrase->SetRuleCount(ruleCountString, scoreVector); + } //targetPhrase->SetDebugOutput(string("New Format pt ") + line); - + // component score, for n-best output std::transform(scoreVector.begin(),scoreVector.end(),scoreVector.begin(),TransformScore); std::transform(scoreVector.begin(),scoreVector.end(),scoreVector.begin(),FloorScore); @@ -238,7 +248,6 @@ bool RuleTableLoaderStandard::Load(FormatType format else { // do nothing } - } // sort and prune each target phrase collection diff --git a/moses/src/RuleTable/PhraseDictionaryALSuffixArray.cpp b/moses/src/RuleTable/PhraseDictionaryALSuffixArray.cpp index 68e2416e7..21acbcab0 100644 --- a/moses/src/RuleTable/PhraseDictionaryALSuffixArray.cpp +++ b/moses/src/RuleTable/PhraseDictionaryALSuffixArray.cpp @@ -64,7 +64,8 @@ void PhraseDictionaryALSuffixArray::InitializeForInput(InputType const& source) std::auto_ptr<RuleTableLoader> loader = RuleTableLoaderFactory::Create(grammarFile); - bool ret = loader->Load(*m_input, *m_output, inFile, *m_weight, m_tableLimit, + std::vector<float> weightT = StaticData::Instance().GetWeights(GetFeature()); + bool ret = loader->Load(*m_input, *m_output, inFile, weightT, m_tableLimit, *m_languageModels, m_wpProducer, *this); CHECK(ret); diff --git a/moses/src/RuleTable/PhraseDictionaryOnDisk.cpp b/moses/src/RuleTable/PhraseDictionaryOnDisk.cpp index 4ae860ec9..9b0124605 100644 --- a/moses/src/RuleTable/PhraseDictionaryOnDisk.cpp +++ b/moses/src/RuleTable/PhraseDictionaryOnDisk.cpp @@ -38,7 +38,7 @@ PhraseDictionaryOnDisk::~PhraseDictionaryOnDisk() bool PhraseDictionaryOnDisk::Load(const std::vector<FactorType> &input , const std::vector<FactorType> &output , const std::string &filePath - , const std::vector<float> &weight + , const std::vector<float> &weight , size_t tableLimit , const LMList& languageModels , const WordPenaltyProducer* wpProducer) @@ -52,8 +52,6 @@ bool PhraseDictionaryOnDisk::Load(const std::vector<FactorType> &input m_inputFactorsVec = input; m_outputFactorsVec = output; - m_weight = weight; - LoadTargetLookup(); if (!m_dbWrapper.BeginLoad(filePath)) @@ -67,6 +65,8 @@ bool PhraseDictionaryOnDisk::Load(const std::vector<FactorType> &input return true; } +// PhraseDictionary impl + //! find list of translations that can translates src. Only for phrase input const TargetPhraseCollection *PhraseDictionaryOnDisk::GetTargetPhraseCollection(const Phrase& /* src */) const { @@ -96,8 +96,7 @@ ChartRuleLookupManager *PhraseDictionaryOnDisk::CreateRuleLookupManager( return new ChartRuleLookupManagerOnDisk(sentence, cellCollection, *this, m_dbWrapper, m_languageModels, m_wpProducer, m_inputFactorsVec, - m_outputFactorsVec, m_weight, - m_filePath); + m_outputFactorsVec, m_filePath); } } diff --git a/moses/src/RuleTable/PhraseDictionaryOnDisk.h b/moses/src/RuleTable/PhraseDictionaryOnDisk.h index 7a09d0e24..f9dcf3a8f 100644 --- a/moses/src/RuleTable/PhraseDictionaryOnDisk.h +++ b/moses/src/RuleTable/PhraseDictionaryOnDisk.h @@ -47,7 +47,6 @@ protected: const LMList* m_languageModels; const WordPenaltyProducer* m_wpProducer; std::vector<FactorType> m_inputFactorsVec, m_outputFactorsVec; - std::vector<float> m_weight; std::string m_filePath; void LoadTargetLookup(); @@ -65,7 +64,7 @@ public: bool Load(const std::vector<FactorType> &input , const std::vector<FactorType> &output , const std::string &filePath - , const std::vector<float> &weight + , const std::vector<float> &weight , size_t tableLimit, const LMList& languageModels, const WordPenaltyProducer* wpProducer); diff --git a/moses/src/ScoreComponentCollection.cpp b/moses/src/ScoreComponentCollection.cpp index fee7e62ea..d0c1e11ed 100644 --- a/moses/src/ScoreComponentCollection.cpp +++ b/moses/src/ScoreComponentCollection.cpp @@ -1,36 +1,201 @@ // $Id$ +#include <vector> #include "ScoreComponentCollection.h" #include "StaticData.h" +using namespace std; + namespace Moses { -ScoreComponentCollection::ScoreComponentCollection() - : m_scores(StaticData::Instance().GetTotalScoreComponents(), 0.0f) - , m_sim(&StaticData::Instance().GetScoreIndexManager()) + +ScoreComponentCollection::ScoreIndexMap ScoreComponentCollection::s_scoreIndexes; +size_t ScoreComponentCollection::s_denseVectorSize = 0; + +ScoreComponentCollection::ScoreComponentCollection() : m_scores(s_denseVectorSize) {} + +void ScoreComponentCollection::RegisterScoreProducer + (const ScoreProducer* scoreProducer) +{ + CHECK(scoreProducer->GetNumScoreComponents() != ScoreProducer::unlimited); + size_t start = s_denseVectorSize; + size_t end = start + scoreProducer->GetNumScoreComponents(); + VERBOSE(1, "ScoreProducer: " << scoreProducer->GetScoreProducerDescription() << " start: " << start << " end: " << end << endl); + s_scoreIndexes[scoreProducer] = pair<size_t,size_t>(start,end); + s_denseVectorSize = end; +} + +void ScoreComponentCollection::UnregisterScoreProducer + (const ScoreProducer* scoreProducer) +{ + CHECK(scoreProducer->GetNumScoreComponents() != ScoreProducer::unlimited); + ScoreIndexMap::iterator iter = s_scoreIndexes.find(scoreProducer); + CHECK(iter != s_scoreIndexes.end()); + s_scoreIndexes.erase(iter); +} + float ScoreComponentCollection::GetWeightedScore() const { - float ret = InnerProduct(StaticData::Instance().GetAllWeights()); - return ret; + return m_scores.inner_product(StaticData::Instance().GetAllWeights().m_scores); } void ScoreComponentCollection::ZeroAllLM(const LMList& lmList) { - - for (size_t ind = lmList.GetMinIndex(); ind <= lmList.GetMaxIndex(); ++ind) { - m_scores[ind] = 0; + for (LMList::const_iterator i = lmList.begin(); i != lmList.end(); ++i) { + Assign(*i, 0); } } void ScoreComponentCollection::PlusEqualsAllLM(const LMList& lmList, const ScoreComponentCollection& rhs) { + for (LMList::const_iterator i = lmList.begin(); i != lmList.end(); ++i) { + PlusEquals(*i,rhs); + } +} + +void ScoreComponentCollection::MultiplyEquals(float scalar) +{ + m_scores *= scalar; +} + +// Multiply all weights of this sparse producer by a given scalar +void ScoreComponentCollection::MultiplyEquals(const ScoreProducer* sp, float scalar) { + assert(sp->GetNumScoreComponents() == ScoreProducer::unlimited); + std::string prefix = sp->GetScoreProducerDescription() + FName::SEP; + for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) { + std::stringstream name; + name << i->first; + if (name.str().substr( 0, prefix.length() ).compare( prefix ) == 0) + m_scores[i->first] = i->second * scalar; + } +} + +// Count weights belonging to this sparse producer +size_t ScoreComponentCollection::GetNumberWeights(const ScoreProducer* sp) { + assert(sp->GetNumScoreComponents() == ScoreProducer::unlimited); + std::string prefix = sp->GetScoreProducerDescription() + FName::SEP; + size_t weights = 0; + for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) { + std::stringstream name; + name << i->first; + if (name.str().substr( 0, prefix.length() ).compare( prefix ) == 0) + weights++; + } + return weights; +} + +void ScoreComponentCollection::DivideEquals(float scalar) +{ + m_scores /= scalar; +} + +void ScoreComponentCollection::CoreDivideEquals(float scalar) +{ + m_scores.coreDivideEquals(scalar); +} + +void ScoreComponentCollection::DivideEquals(const ScoreComponentCollection& rhs) +{ + m_scores.divideEquals(rhs.m_scores); +} + +void ScoreComponentCollection::MultiplyEquals(const ScoreComponentCollection& rhs) +{ + m_scores *= rhs.m_scores; +} + +void ScoreComponentCollection::MultiplyEqualsBackoff(const ScoreComponentCollection& rhs, float backoff) +{ + m_scores.multiplyEqualsBackoff(rhs.m_scores, backoff); +} + +void ScoreComponentCollection::MultiplyEquals(float core_r0, float sparse_r0) +{ + m_scores.multiplyEquals(core_r0, sparse_r0); +} + +std::ostream& operator<<(std::ostream& os, const ScoreComponentCollection& rhs) +{ + os << rhs.m_scores; + return os; +} +void ScoreComponentCollection::L1Normalise() { + m_scores /= m_scores.l1norm_coreFeatures(); +} + +float ScoreComponentCollection::GetL1Norm() const { + return m_scores.l1norm(); +} + +float ScoreComponentCollection::GetL2Norm() const { + return m_scores.l2norm(); +} - for (size_t ind = lmList.GetMinIndex(); ind <= lmList.GetMaxIndex(); ++ind) { - m_scores[ind] += rhs.m_scores[ind]; +float ScoreComponentCollection::GetLInfNorm() const { + return m_scores.linfnorm(); +} + +size_t ScoreComponentCollection::L1Regularize(float lambda) { + return m_scores.l1regularize(lambda); +} + +void ScoreComponentCollection::L2Regularize(float lambda) { + m_scores.l2regularize(lambda); +} + +size_t ScoreComponentCollection::SparseL1Regularize(float lambda) { + return m_scores.sparseL1regularize(lambda); +} + +void ScoreComponentCollection::SparseL2Regularize(float lambda) { + m_scores.sparseL2regularize(lambda); +} + +void ScoreComponentCollection::Save(ostream& out) const { + ScoreIndexMap::const_iterator iter = s_scoreIndexes.begin(); + for (; iter != s_scoreIndexes.end(); ++iter ) { + string name = iter->first->GetScoreProducerDescription(); + IndexPair ip = iter->second; // feature indices + if (ip.second-ip.first == 1) { + out << name << " " << m_scores[ip.first] << endl; + } else { + for (size_t i=ip.first; i < ip.second; ++i) { + ostringstream fullname; + fullname << name << "_" << (i + 1 - ip.first); + out << fullname.str() << " " << m_scores[i] << endl; + } + } + } + + // write sparse features + m_scores.write(out); +} + +void ScoreComponentCollection::Save(const string& filename) const { + ofstream out(filename.c_str()); + if (!out) { + ostringstream msg; + msg << "Unable to open " << filename; + throw runtime_error(msg.str()); } + Save(out); + out.close(); +} +void ScoreComponentCollection::Assign(const ScoreProducer* sp, const string line) { + CHECK(sp->GetNumScoreComponents() == ScoreProducer::unlimited); + istringstream istr(line); + while(istr) { + string namestring; + FValue value; + istr >> namestring; + if (!istr) break; + istr >> value; + FName fname(sp->GetScoreProducerDescription(), namestring); + m_scores[fname] = value; + } } } diff --git a/moses/src/ScoreComponentCollection.h b/moses/src/ScoreComponentCollection.h index 101788d81..3b9ff1d08 100644 --- a/moses/src/ScoreComponentCollection.h +++ b/moses/src/ScoreComponentCollection.h @@ -23,17 +23,26 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #define moses_ScoreComponentCollection_h #include <numeric> +#include <sstream> + +#ifdef MPI_ENABLE +#include <boost/serialization/access.hpp> +#include <boost/serialization/split_member.hpp> +#endif + #include "util/check.hh" #include "LMList.h" #include "ScoreProducer.h" -#include "ScoreIndexManager.h" +#include "FeatureVector.h" #include "TypeDef.h" #include "Util.h" + namespace Moses { + /*** An unweighted collection of scores for a translation or step in a translation. * * In the factored phrase-based models that are implemented by moses, there are a set of @@ -55,158 +64,374 @@ namespace Moses class ScoreComponentCollection { friend std::ostream& operator<<(std::ostream& os, const ScoreComponentCollection& rhs); - friend class ScoreIndexManager; private: - std::vector<float> m_scores; - const ScoreIndexManager* m_sim; + FVector m_scores; + typedef std::pair<size_t,size_t> IndexPair; + typedef std::map<const ScoreProducer*,IndexPair> ScoreIndexMap; + static ScoreIndexMap s_scoreIndexes; + static size_t s_denseVectorSize; + static IndexPair GetIndexes(const ScoreProducer* sp) + { + ScoreIndexMap::const_iterator indexIter = s_scoreIndexes.find(sp); + if (indexIter == s_scoreIndexes.end()) { + std::cerr << "ERROR: ScoreProducer: " << sp->GetScoreProducerDescription() << + " not registered with ScoreIndexMap" << std::endl; + std::cerr << "You must call ScoreComponentCollection.RegisterScoreProducer() " << + " for every ScoreProducer" << std::endl; + abort(); + } + return indexIter->second; + } public: + static void ResetCounter() { + s_denseVectorSize = 0; + } + //! Create a new score collection with all values set to 0.0 ScoreComponentCollection(); //! Clone a score collection - ScoreComponentCollection(const ScoreComponentCollection& rhs) - : m_scores(rhs.m_scores) - , m_sim(rhs.m_sim) - {} + ScoreComponentCollection(const ScoreComponentCollection& rhs) + : m_scores(rhs.m_scores) + {} + + /** + * Register a ScoreProducer with a fixed number of scores, so that it can + * be allocated space in the dense part of the feature vector. + **/ + static void RegisterScoreProducer(const ScoreProducer* scoreProducer); + static void UnregisterScoreProducer(const ScoreProducer* scoreProducer); - inline size_t size() const { - return m_scores.size(); + /** Load from file */ + bool Load(const std::string& filename) + { + return m_scores.load(filename); } - const float& operator[](size_t x) const { - return m_scores[x]; + + const FVector& GetScoresVector() const + { + return m_scores; } - //! Set all values to 0.0 - void ZeroAll() { - for (std::vector<float>::iterator i=m_scores.begin(); i!=m_scores.end(); ++i) - *i = 0.0f; + size_t Size() const + { + return m_scores.size(); } - //! add the score in rhs - void PlusEquals(const ScoreComponentCollection& rhs) { - CHECK(m_scores.size() >= rhs.m_scores.size()); - const size_t l = rhs.m_scores.size(); - for (size_t i=0; i<l; i++) { - m_scores[i] += rhs.m_scores[i]; + void Resize() + { + if (m_scores.coreSize() != s_denseVectorSize) { + m_scores.resize(s_denseVectorSize); } } - //! subtract the score in rhs - void MinusEquals(const ScoreComponentCollection& rhs) { - CHECK(m_scores.size() >= rhs.m_scores.size()); - const size_t l = rhs.m_scores.size(); - for (size_t i=0; i<l; i++) { - m_scores[i] -= rhs.m_scores[i]; - } + /** Create and FVector with the right number of core features */ + static FVector CreateFVector() + { + return FVector(s_denseVectorSize); } - //! Add scores from a single ScoreProducer only - //! The length of scores must be equal to the number of score components - //! produced by sp - void PlusEquals(const ScoreProducer* sp, const std::vector<float>& scores) { - CHECK(scores.size() == sp->GetNumScoreComponents()); - size_t i = m_sim->GetBeginIndex(sp->GetScoreBookkeepingID()); - for (std::vector<float>::const_iterator vi = scores.begin(); - vi != scores.end(); ++vi) { - m_scores[i++] += *vi; - } + void SetToBinaryOf(const ScoreComponentCollection& rhs) + { + m_scores.setToBinaryOf(rhs.m_scores); + } + + //! Set all values to 0.0 + void ZeroAll() + { + m_scores.clear(); + } + + void MultiplyEquals(float scalar); + void DivideEquals(float scalar); + void CoreDivideEquals(float scalar); + void DivideEquals(const ScoreComponentCollection& rhs); + void MultiplyEquals(const ScoreComponentCollection& rhs); + void MultiplyEqualsBackoff(const ScoreComponentCollection& rhs, float backoff); + void MultiplyEquals(float core_r0, float sparse_r0); + void MultiplyEquals(const ScoreProducer* sp, float scalar); + + size_t GetNumberWeights(const ScoreProducer* sp); + + void CoreAssign(const ScoreComponentCollection& rhs) + { + m_scores.coreAssign(rhs.m_scores); + } + + //! add the score in rhs + void PlusEquals(const ScoreComponentCollection& rhs) + { + m_scores += rhs.m_scores; + } + + // add only sparse features + void SparsePlusEquals(const ScoreComponentCollection& rhs) + { + m_scores.sparsePlusEquals(rhs.m_scores); + } + + void PlusEquals(const FVector& scores) + { + m_scores += scores; + } + + //! subtract the score in rhs + void MinusEquals(const ScoreComponentCollection& rhs) + { + m_scores -= rhs.m_scores; + } + + //For features which have an unbounded number of components + void MinusEquals(const ScoreProducer*sp, const std::string& name, float score) + { + assert(sp->GetNumScoreComponents() == ScoreProducer::unlimited); + FName fname(sp->GetScoreProducerDescription(),name); + m_scores[fname] -= score; + } + + //For features which have an unbounded number of components + void SparseMinusEquals(const std::string& full_name, float score) + { + FName fname(full_name); + m_scores[fname] -= score; } - //! Add scores from a single ScoreProducer only - //! The length of scores must be equal to the number of score components - //! produced by sp - void PlusEquals(const ScoreProducer* sp, const ScoreComponentCollection& scores) { - size_t i = m_sim->GetBeginIndex(sp->GetScoreBookkeepingID()); - const size_t end = m_sim->GetEndIndex(sp->GetScoreBookkeepingID()); - for (; i < end; ++i) { + //! Add scores from a single ScoreProducer only + //! The length of scores must be equal to the number of score components + //! produced by sp + void PlusEquals(const ScoreProducer* sp, const ScoreComponentCollection& scores) + { + IndexPair indexes = GetIndexes(sp); + for (size_t i = indexes.first; i < indexes.second; ++i) { m_scores[i] += scores.m_scores[i]; } - } + } - //! Special version PlusEquals(ScoreProducer, vector<float>) - //! to add the score from a single ScoreProducer that produces - //! a single value - void PlusEquals(const ScoreProducer* sp, float score) { - CHECK(1 == sp->GetNumScoreComponents()); - const size_t i = m_sim->GetBeginIndex(sp->GetScoreBookkeepingID()); - m_scores[i] += score; - } - - void Assign(const ScoreProducer* sp, const std::vector<float>& scores) { - CHECK(scores.size() == sp->GetNumScoreComponents()); - size_t i = m_sim->GetBeginIndex(sp->GetScoreBookkeepingID()); - for (std::vector<float>::const_iterator vi = scores.begin(); - vi != scores.end(); ++vi) { - m_scores[i++] = *vi; + //! Add scores from a single ScoreProducer only + //! The length of scores must be equal to the number of score components + //! produced by sp + void PlusEquals(const ScoreProducer* sp, const std::vector<float>& scores) + { + IndexPair indexes = GetIndexes(sp); + CHECK(scores.size() == indexes.second - indexes.first); + for (size_t i = 0; i < scores.size(); ++i) { + m_scores[i + indexes.first] += scores[i]; } } - void Assign(const ScoreComponentCollection ©) { - m_scores = copy.m_scores; + //! Special version PlusEquals(ScoreProducer, vector<float>) + //! to add the score from a single ScoreProducer that produces + //! a single value + void PlusEquals(const ScoreProducer* sp, float score) + { + IndexPair indexes = GetIndexes(sp); + CHECK(1 == indexes.second - indexes.first); + m_scores[indexes.first] += score; + } + + //For features which have an unbounded number of components + void PlusEquals(const ScoreProducer*sp, const std::string& name, float score) + { + CHECK(sp->GetNumScoreComponents() == ScoreProducer::unlimited); + FName fname(sp->GetScoreProducerDescription(),name); + m_scores[fname] += score; + } + + //For features which have an unbounded number of components + void SparsePlusEquals(const std::string& full_name, float score) + { + FName fname(full_name); + m_scores[fname] += score; } - //! Special version PlusEquals(ScoreProducer, vector<float>) + void Assign(const ScoreProducer* sp, const std::vector<float>& scores) + { + IndexPair indexes = GetIndexes(sp); + CHECK(scores.size() == indexes.second - indexes.first); + for (size_t i = 0; i < scores.size(); ++i) { + m_scores[i + indexes.first] = scores[i]; + } + } + + //! Special version Assign(ScoreProducer, vector<float>) //! to add the score from a single ScoreProducer that produces //! a single value - void Assign(const ScoreProducer* sp, float score) { - CHECK(1 == sp->GetNumScoreComponents()); - const size_t i = m_sim->GetBeginIndex(sp->GetScoreBookkeepingID()); - m_scores[i] = score; + void Assign(const ScoreProducer* sp, float score) + { + IndexPair indexes = GetIndexes(sp); + CHECK(1 == indexes.second - indexes.first); + m_scores[indexes.first] = score; + } + + // Assign core weight by index + void Assign(size_t index, float score) { + m_scores[index] = score; } - //! Used to find the weighted total of scores. rhs should contain a vector of weights - //! of the same length as the number of scores. - float InnerProduct(const std::vector<float>& rhs) const { - return std::inner_product(m_scores.begin(), m_scores.end(), rhs.begin(), 0.0f); + //For features which have an unbounded number of components + void Assign(const ScoreProducer*sp, const std::string name, float score) + { + CHECK(sp->GetNumScoreComponents() == ScoreProducer::unlimited); + FName fname(sp->GetScoreProducerDescription(),name); + m_scores[fname] = score; } - float PartialInnerProduct(const ScoreProducer* sp, const std::vector<float>& rhs) const { - std::vector<float> lhs = GetScoresForProducer(sp); - CHECK(lhs.size() == rhs.size()); - return std::inner_product(lhs.begin(), lhs.end(), rhs.begin(), 0.0f); + + //Read sparse features from string + void Assign(const ScoreProducer* sp, const std::string line); + + // shortcut: setting the value directly using the feature name + void Assign(const std::string name, float score) + { + FName fname(name); + m_scores[fname] = score; } - //! return a vector of all the scores associated with a certain ScoreProducer - std::vector<float> GetScoresForProducer(const ScoreProducer* sp) const { - size_t id = sp->GetScoreBookkeepingID(); - const size_t begin = m_sim->GetBeginIndex(id); - const size_t end = m_sim->GetEndIndex(id); - std::vector<float> res(end-begin); - size_t j = 0; - for (size_t i = begin; i < end; i++) { - res[j++] = m_scores[i]; + float InnerProduct(const ScoreComponentCollection& rhs) const + { + return m_scores.inner_product(rhs.m_scores); + } + + float PartialInnerProduct(const ScoreProducer* sp, const std::vector<float>& rhs) const + { + std::vector<float> lhs = GetScoresForProducer(sp); + CHECK(lhs.size() == rhs.size()); + return std::inner_product(lhs.begin(), lhs.end(), rhs.begin(), 0.0f); + } + + //! return a vector of all the scores associated with a certain ScoreProducer + std::vector<float> GetScoresForProducer(const ScoreProducer* sp) const + { + size_t components = sp->GetNumScoreComponents(); + if (components == ScoreProducer::unlimited) return std::vector<float>(); + std::vector<float> res(components); + IndexPair indexes = GetIndexes(sp); + for (size_t i = 0; i < res.size(); ++i) { + res[i] = m_scores[i + indexes.first]; } return res; + } + + //! get subset of scores that belong to a certain sparse ScoreProducer + FVector GetVectorForProducer(const ScoreProducer* sp) const + { + FVector fv(s_denseVectorSize); + std::string prefix = sp->GetScoreProducerDescription() + FName::SEP; + for(FVector::FNVmap::const_iterator i = m_scores.cbegin(); i != m_scores.cend(); i++) { + std::stringstream name; + name << i->first; + if (name.str().substr( 0, prefix.length() ).compare( prefix ) == 0) + fv[i->first] = i->second; + } + return fv; } - //! if a ScoreProducer produces a single score (for example, a language model score) - //! this will return it. If not, this method will throw - float GetScoreForProducer(const ScoreProducer* sp) const { - size_t id = sp->GetScoreBookkeepingID(); - const size_t begin = m_sim->GetBeginIndex(id); -#ifndef NDEBUG - const size_t end = m_sim->GetEndIndex(id); - CHECK(end-begin == 1); -#endif - return m_scores[begin]; + float GetSparseWeight(const FName& featureName) const + { + return m_scores[featureName]; + } + + void PrintCoreFeatures() { + m_scores.printCoreFeatures(); + } + + void ThresholdScaling(float maxValue) + { + // find (smallest) factor for which all weights are <= maxValue + // 0.1 / 0.14 = 0.714285714 + // 0.1 / 0.17 = 0.588235294 + m_scores.thresholdScale(maxValue); + } + + void CapMax(float maxValue) + { + // cap all sparse features to maxValue + m_scores.capMax(maxValue); + } + + void CapMin(float minValue) + { + // cap all sparse features to minValue + m_scores.capMin(minValue); + } + + //! if a ScoreProducer produces a single score (for example, a language model score) + //! this will return it. If not, this method will throw + float GetScoreForProducer(const ScoreProducer* sp) const + { + IndexPair indexes = GetIndexes(sp); + CHECK(indexes.second - indexes.first == 1); + return m_scores[indexes.first]; + } + + //For features which have an unbounded number of components + float GetScoreForProducer + (const ScoreProducer* sp, const std::string& name) const + { + CHECK(sp->GetNumScoreComponents() == ScoreProducer::unlimited); + FName fname(sp->GetScoreProducerDescription(),name); + return m_scores[fname]; } - float GetWeightedScore() const; + float GetWeightedScore() const; void ZeroAllLM(const LMList& lmList); void PlusEqualsAllLM(const LMList& lmList, const ScoreComponentCollection& rhs); + void L1Normalise(); + float GetL1Norm() const; + float GetL2Norm() const; + float GetLInfNorm() const; + size_t L1Regularize(float lambda); + void L2Regularize(float lambda); + size_t SparseL1Regularize(float lambda); + void SparseL2Regularize(float lambda); + void Save(const std::string& filename) const; + void Save(std::ostream&) const; + + void IncrementSparseHopeFeatures() { m_scores.incrementSparseHopeFeatures(); } + void IncrementSparseFearFeatures() { m_scores.incrementSparseFearFeatures(); } + void PrintSparseHopeFeatureCounts(std::ofstream& out) { m_scores.printSparseHopeFeatureCounts(out); } + void PrintSparseFearFeatureCounts(std::ofstream& out) { m_scores.printSparseFearFeatureCounts(out); } + void PrintSparseHopeFeatureCounts() { m_scores.printSparseHopeFeatureCounts(); } + void PrintSparseFearFeatureCounts() { m_scores.printSparseFearFeatureCounts(); } + size_t PruneSparseFeatures(size_t threshold) { return m_scores.pruneSparseFeatures(threshold); } + size_t PruneZeroWeightFeatures() { return m_scores.pruneZeroWeightFeatures(); } + void UpdateConfidenceCounts(ScoreComponentCollection &weightUpdate, bool signedCounts) { m_scores.updateConfidenceCounts(weightUpdate.m_scores, signedCounts); } + void UpdateLearningRates(float decay_core, float decay_sparse, ScoreComponentCollection &confidenceCounts, float core_r0, float sparse_r0) { m_scores.updateLearningRates(decay_core, decay_sparse, confidenceCounts.m_scores, core_r0, sparse_r0); } -}; +#ifdef MPI_ENABLE + public: + friend class boost::serialization::access; + + private: + //serialization + template<class Archive> + void save(Archive &ar, const unsigned int version) const { + ar << m_scores; + } + + template<class Archive> + void load(Archive &ar, const unsigned int version) { + ar >> m_scores; -inline std::ostream& operator<<(std::ostream& os, const ScoreComponentCollection& rhs) -{ - os << "<<" << rhs.m_scores[0]; - for (size_t i=1; i<rhs.m_scores.size(); i++) - os << ", " << rhs.m_scores[i]; - return os << ">>"; -} + } + + BOOST_SERIALIZATION_SPLIT_MEMBER() + +#endif +}; + +struct SCCPlus { + ScoreComponentCollection operator() + (const ScoreComponentCollection& lhs, + const ScoreComponentCollection& rhs) { + ScoreComponentCollection sum(lhs); + sum.PlusEquals(rhs); + return sum; + } +}; } #endif diff --git a/moses/src/ScoreComponentCollectionTest.cpp b/moses/src/ScoreComponentCollectionTest.cpp new file mode 100644 index 000000000..10e96a017 --- /dev/null +++ b/moses/src/ScoreComponentCollectionTest.cpp @@ -0,0 +1,141 @@ +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2010 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#include <stdexcept> + +#include <boost/test/unit_test.hpp> + +#include "DummyScoreProducers.h" +#include "FeatureFunction.h" +#include "ScoreComponentCollection.h" + +using namespace Moses; +using namespace std; + +BOOST_AUTO_TEST_SUITE(scc) + +class MockSingleFeature : public StatelessFeatureFunction { + public: + MockSingleFeature(): StatelessFeatureFunction("MockSingle",1) {} + std::string GetScoreProducerWeightShortName(unsigned) const {return "sf";} +}; + +class MockMultiFeature : public StatelessFeatureFunction { + public: + MockMultiFeature(): StatelessFeatureFunction("MockMulti", 5) {} + std::string GetScoreProducerWeightShortName(unsigned) const {return "mf";} +}; + +class MockSparseFeature : public StatelessFeatureFunction { + public: + MockSparseFeature(): StatelessFeatureFunction("MockSparse", ScoreProducer::unlimited) {} + std::string GetScoreProducerWeightShortName(unsigned) const {return "sf";} +}; + + +struct MockProducers { + MockProducers() {} + + MockSingleFeature single; + MockMultiFeature multi; + MockSparseFeature sparse; +}; + +BOOST_FIXTURE_TEST_CASE(ctor, MockProducers) +{ + ScoreComponentCollection scc; + BOOST_CHECK_EQUAL(scc.GetScoreForProducer(&single),0); + float expected[] = {0,0,0,0,0}; + std::vector<float> actual= scc.GetScoresForProducer(&multi); + BOOST_CHECK_EQUAL_COLLECTIONS(expected, expected+5, actual.begin(), actual.begin()+5); +} + +BOOST_FIXTURE_TEST_CASE(plusequals, MockProducers) +{ + float arr1[] = {1,2,3,4,5}; + float arr2[] = {2,4,6,8,10}; + std::vector<float> vec1(arr1,arr1+5); + std::vector<float> vec2(arr2,arr2+5); + + ScoreComponentCollection scc; + scc.PlusEquals(&single, 3.4f); + BOOST_CHECK_EQUAL(scc.GetScoreForProducer(&single), 3.4f); + scc.PlusEquals(&multi,vec1); + std::vector<float> actual = scc.GetScoresForProducer(&multi); + BOOST_CHECK_EQUAL_COLLECTIONS(vec1.begin(),vec1.end() + ,actual.begin(), actual.end()); + scc.PlusEquals(&multi,vec1); + actual = scc.GetScoresForProducer(&multi); + BOOST_CHECK_EQUAL_COLLECTIONS(vec2.begin(),vec2.end(), + actual.begin(), actual.end()); + + BOOST_CHECK_EQUAL(scc.GetScoreForProducer(&single), 3.4f); +} + +BOOST_FIXTURE_TEST_CASE(sparse_feature, MockProducers) +{ + ScoreComponentCollection scc; + scc.Assign(&sparse, "first", 1.3f); + scc.Assign(&sparse, "second", 2.1f); + BOOST_CHECK_EQUAL( scc.GetScoreForProducer(&sparse,"first"), 1.3f); + BOOST_CHECK_EQUAL( scc.GetScoreForProducer(&sparse,"second"), 2.1f); + BOOST_CHECK_EQUAL( scc.GetScoreForProducer(&sparse,"third"), 0.0f); + scc.Assign(&sparse, "first", -1.9f); + BOOST_CHECK_EQUAL( scc.GetScoreForProducer(&sparse,"first"), -1.9f); + scc.PlusEquals(&sparse, "first", -1.9f); + BOOST_CHECK_EQUAL( scc.GetScoreForProducer(&sparse,"first"), -3.8f); +} + +/* + Doesn't work because of the static registration of ScoreProducers + in ScoreComponentCollection. +BOOST_FIXTURE_TEST_CASE(save, MockProducers) +{ + ScoreComponentCollection scc; + scc.Assign(&sparse, "first", 1.1f); + scc.Assign(&single, 0.25f); + float arr[] = {1,2.1,3,4,5}; + std::vector<float> vec1(arr,arr+5); + scc.Assign(&multi,vec1); + ostringstream out; + scc.Save(out); + cerr << out.str() << endl; + istringstream in (out.str()); + string line; + getline(in,line); + BOOST_CHECK_EQUAL(line, "MockSingle:4_1 0.25"); + getline(in,line); + BOOST_CHECK_EQUAL(line, "MockMulti:4_1 1"); + getline(in,line); + BOOST_CHECK_EQUAL(line, "MockMulti:4_2 2.1"); + getline(in,line); + BOOST_CHECK_EQUAL(line, "MockMulti:4_3 3"); + getline(in,line); + BOOST_CHECK_EQUAL(line, "MockMulti:4_4 4"); + getline(in,line); + BOOST_CHECK_EQUAL(line, "MockMulti:4_5 5"); + getline(in,line); + BOOST_CHECK_EQUAL(line,"MockSparse:4_first 1.1"); + BOOST_CHECK(!getline(in,line)); +} +*/ + + +BOOST_AUTO_TEST_SUITE_END() + diff --git a/moses/src/ScoreIndexManager.cpp b/moses/src/ScoreIndexManager.cpp deleted file mode 100644 index 8378a668a..000000000 --- a/moses/src/ScoreIndexManager.cpp +++ /dev/null @@ -1,143 +0,0 @@ - // $Id$ - -#include <iostream> -#include <iomanip> -#include <string> -#include <cstdio> -#include "util/check.hh" -#include "Util.h" -#include "StaticData.h" -#include "ScoreIndexManager.h" -#include "ScoreProducer.h" -#include "ScoreComponentCollection.h" // debugging - -namespace Moses -{ -using namespace std; - -void ScoreIndexManager::AddScoreProducer(const ScoreProducer* sp) -{ - // Producers must be inserted in the order they are created - const_cast<ScoreProducer*>(sp)->CreateScoreBookkeepingID(); - CHECK(m_begins.size() == (sp->GetScoreBookkeepingID())); - - m_producers.push_back(sp); - - m_begins.push_back(m_last); - size_t numScoreCompsProduced = sp->GetNumScoreComponents(); - CHECK(numScoreCompsProduced > 0); - m_last += numScoreCompsProduced; - m_ends.push_back(m_last); - VERBOSE(3,"Added ScoreProducer(" << sp->GetScoreBookkeepingID() - << " " << sp->GetScoreProducerDescription() - << ") index=" << m_begins.back() << "-" << m_ends.back()-1 << std::endl); - -} - -void ScoreIndexManager::PrintLabeledScores(std::ostream& os, const ScoreComponentCollection& scores) const -{ - std::vector<float> weights(scores.m_scores.size(), 1.0f); - PrintLabeledWeightedScores(os, scores, weights); -} - -void ScoreIndexManager::PrintLabeledWeightedScores(std::ostream& os, const ScoreComponentCollection& scores, const std::vector<float>& weights) const -{ - CHECK(m_featureShortNames.size() == weights.size()); - string lastName = ""; - for (size_t i = 0; i < m_featureShortNames.size(); ++i) { - if (i>0) { - os << " "; - } - if (lastName != m_featureShortNames[i]) { - os << m_featureShortNames[i] << ": "; - lastName = m_featureShortNames[i]; - } - os << weights[i] * scores[i]; - } -} - -void ScoreIndexManager::InitFeatureNames() -{ - m_featureNames.clear(); - m_featureShortNames.clear(); - size_t cur_i = 0; - size_t cur_scoreType = 0; - while (cur_i < m_last) { - size_t nis_idx = 0; - bool add_idx = (m_producers[cur_scoreType]->GetNumInputScores() > 1); - while (nis_idx < m_producers[cur_scoreType]->GetNumInputScores()) { - ostringstream os; - os << m_producers[cur_scoreType]->GetScoreProducerDescription(nis_idx); - if (add_idx) - os << '_' << (nis_idx+1); - m_featureNames.push_back(os.str()); - nis_idx++; - cur_i++; - } - - int ind = 1; - add_idx = (m_ends[cur_scoreType] - cur_i > 1); - while (cur_i < m_ends[cur_scoreType]) { - ostringstream os; - os << m_producers[cur_scoreType]->GetScoreProducerDescription(nis_idx+ind-1); - if (add_idx) - os << '_' << ind; - m_featureNames.push_back(os.str()); - m_featureShortNames.push_back( m_producers[cur_scoreType]->GetScoreProducerWeightShortName(nis_idx+ind-1) ); - ++cur_i; - ++ind; - } - cur_scoreType++; - } -} - -#ifdef HAVE_PROTOBUF -void ScoreIndexManager::SerializeFeatureNamesToPB(hgmert::Hypergraph* hg) const -{ - for (size_t i = 0; i < m_featureNames.size(); ++i) { - hg->add_feature_names(m_featureNames[i]); - } -} -#endif - -void ScoreIndexManager::InitWeightVectorFromFile(const std::string& fnam, vector<float>* m_allWeights) const -{ - CHECK(m_allWeights->size() == m_featureNames.size()); - ifstream in(fnam.c_str()); - CHECK(in.good()); - char buf[2000]; - map<string, double> name2val; - while (!in.eof()) { - in.getline(buf, 2000); - if (strlen(buf) == 0) continue; - if (buf[0] == '#') continue; - istringstream is(buf); - string fname; - double val; - is >> fname >> val; - map<string, double>::iterator i = name2val.find(fname); - CHECK(i == name2val.end()); // duplicate weight name - name2val[fname] = val; - } - CHECK(m_allWeights->size() == m_featureNames.size()); - for (size_t i = 0; i < m_featureNames.size(); ++i) { - map<string, double>::iterator iter = name2val.find(m_featureNames[i]); - if (iter == name2val.end()) { - cerr << "No weight found found for feature: " << m_featureNames[i] << endl; - abort(); - } - (*m_allWeights)[i] = iter->second; - } -} - -std::ostream& operator<<(std::ostream& os, const ScoreIndexManager& sim) -{ - for (size_t i = 0; i < sim.m_featureNames.size(); ++i) { - os << sim.m_featureNames[i] << endl; - } - os << endl; - return os; -} - -} - diff --git a/moses/src/ScoreIndexManager.h b/moses/src/ScoreIndexManager.h deleted file mode 100644 index 9400dc044..000000000 --- a/moses/src/ScoreIndexManager.h +++ /dev/null @@ -1,68 +0,0 @@ -// $Id$ - -#ifndef moses_ScoreIndexManager_h -#define moses_ScoreIndexManager_h - -#include <iostream> -#include <vector> -#ifdef HAVE_PROTOBUF -#include "hypergraph.pb.h" -#endif - -namespace Moses -{ - -class ScoreProducer; -class ScoreComponentCollection; // debugging only -class StatefulFeatureFunction; -class StatelessFeatureFunction; - -/** Keep track of scores and score producers. Each score producer is reserved contiguous set of slots - * to put their score components. All the score components are arranged in a vector with no gaps. - * Only 1 ScoreIndexManager object should be instantiated - */ -class ScoreIndexManager -{ - friend std::ostream& operator<<(std::ostream& os, const ScoreIndexManager& sim); -public: - ScoreIndexManager() : m_last(0) {} - - //! new score producer to manage. Producers must be inserted in the order they are created - void AddScoreProducer(const ScoreProducer* producer); - void InitFeatureNames(); - - //! starting score index for a particular score producer with scoreBookkeepingID - size_t GetBeginIndex(size_t scoreBookkeepingID) const { - return m_begins[scoreBookkeepingID]; - } - //! end score index for a particular score producer with scoreBookkeepingID - size_t GetEndIndex(size_t scoreBookkeepingID) const { - return m_ends[scoreBookkeepingID]; - } - //! sum of all score components from every score producer - size_t GetTotalNumberOfScores() const { - return m_last; - } - //! print unweighted scores of each ScoreManager to stream os - void PrintLabeledScores(std::ostream& os, const ScoreComponentCollection& scc) const; - //! print weighted scores of each ScoreManager to stream os - void PrintLabeledWeightedScores(std::ostream& os, const ScoreComponentCollection& scc, const std::vector<float>& weights) const; -#ifdef HAVE_PROTOBUF - void SerializeFeatureNamesToPB(hgmert::Hypergraph* hg) const; -#endif - void InitWeightVectorFromFile(const std::string& fnam, std::vector<float>* m_allWeights) const; -private: - ScoreIndexManager(const ScoreIndexManager&); // don't implement - - std::vector<size_t> m_begins; - std::vector<size_t> m_ends; - std::vector<const ScoreProducer*> m_producers; /**< all the score producers in this run */ - std::vector<std::string> m_featureNames; - std::vector<std::string> m_featureShortNames; - size_t m_last; -}; - - -} - -#endif diff --git a/moses/src/ScoreProducer.cpp b/moses/src/ScoreProducer.cpp index 092c78e0c..07823125c 100644 --- a/moses/src/ScoreProducer.cpp +++ b/moses/src/ScoreProducer.cpp @@ -1,20 +1,38 @@ // $Id$ #include <iostream> -#include <typeinfo> +#include <sstream> + +#include "ScoreComponentCollection.h" #include "ScoreProducer.h" -#include "StaticData.h" -#include "ScoreIndexManager.h" + +using namespace std; namespace Moses { -unsigned int ScoreProducer::s_globalScoreBookkeepingIdCounter(0); -ScoreProducer::~ScoreProducer() {} +multiset<string> ScoreProducer::description_counts; +const size_t ScoreProducer::unlimited = -1; -ScoreProducer::ScoreProducer() +ScoreProducer::ScoreProducer(const std::string& description, size_t numScoreComponents) + : m_reportSparseFeatures(false), m_numScoreComponents(numScoreComponents) { - m_scoreBookkeepingId = UNASSIGNED; + description_counts.insert(description); + size_t count = description_counts.count(description); + ostringstream dstream; + dstream << description; + if (count > 1) + { + dstream << ":" << count; + } + m_description = dstream.str(); + if (numScoreComponents != unlimited) + { + ScoreComponentCollection::RegisterScoreProducer(this); + } +} + +ScoreProducer::~ScoreProducer() { } } diff --git a/moses/src/ScoreProducer.h b/moses/src/ScoreProducer.h index 0065b774a..c88e5efca 100644 --- a/moses/src/ScoreProducer.h +++ b/moses/src/ScoreProducer.h @@ -3,54 +3,48 @@ #ifndef moses_ScoreProducer_h #define moses_ScoreProducer_h +#include <set> #include <string> -#include <limits> +#include <vector> + +#include "FeatureVector.h" namespace Moses { -class Hypothesis; -class ScoreComponentCollection; -class ScoreIndexManager; -class FFState; - -/** to keep track of the various things that can produce a score, - * we use this evil implementation-inheritance to give them each - * a unique, sequential (read: good for vector indices) ID - * + /* * @note do not confuse this with a producer/consumer pattern. * this is not a producer in that sense. */ class ScoreProducer { private: - static unsigned int s_globalScoreBookkeepingIdCounter; - unsigned int m_scoreBookkeepingId; - - ScoreProducer(const ScoreProducer&); // don't implement - -#define UNASSIGNED std::numeric_limits<unsigned int>::max() - + std::string m_description; + bool m_reportSparseFeatures; + size_t m_numScoreComponents; + //In case there's multiple producers with the same description + static std::multiset<std::string> description_counts; + ScoreProducer(const ScoreProducer&); // don't implement + protected: - // it would be nice to force registration here, but some Producer objects - // are constructed before they know how many scores they have - ScoreProducer(); - virtual ~ScoreProducer(); + ScoreProducer(const std::string& description, size_t numScoreComponents); + virtual ~ScoreProducer(); public: - //! contiguous id - unsigned int GetScoreBookkeepingID() const { - return m_scoreBookkeepingId; - } - void CreateScoreBookkeepingID() { - m_scoreBookkeepingId = s_globalScoreBookkeepingIdCounter++; + + static const size_t unlimited; + + static void ResetDescriptionCounts() { + description_counts.clear(); } - //! returns the number of scores that a subclass produces. - //! For example, a language model conventionally produces 1, a translation table some arbitrary number, etc - virtual size_t GetNumScoreComponents() const = 0; - //! returns a string description of this producer - virtual std::string GetScoreProducerDescription(unsigned idx=0) const = 0; + //! returns the number of scores that a subclass produces. + //! For example, a language model conventionally produces 1, a translation table some arbitrary number, etc + //! sparse features returned unlimited + size_t GetNumScoreComponents() const {return m_numScoreComponents;} + + //! returns a string description of this producer + const std::string& GetScoreProducerDescription() const {return m_description;} //! returns the weight parameter name of this producer (used in n-best list) virtual std::string GetScoreProducerWeightShortName(unsigned idx=0) const = 0; @@ -60,8 +54,12 @@ public: return 0; }; - virtual bool IsStateless() const = 0; + virtual bool IsStateless() const = 0; + + void SetSparseFeatureReporting() { m_reportSparseFeatures = true; } + bool GetSparseFeatureReporting() const { return m_reportSparseFeatures; } + virtual float GetSparseProducerWeight() const { return 1; } }; diff --git a/moses/src/SearchCubePruning.cpp b/moses/src/SearchCubePruning.cpp index 4f98208bb..9fe6aec24 100644 --- a/moses/src/SearchCubePruning.cpp +++ b/moses/src/SearchCubePruning.cpp @@ -145,7 +145,7 @@ void SearchCubePruning::ProcessSentence() stackNo++; } - PrintBitmapContainerGraph(); + //PrintBitmapContainerGraph(); // some more logging IFVERBOSE(2) { diff --git a/moses/src/Sentence.cpp b/moses/src/Sentence.cpp index 2a765fe59..b19e2e591 100644 --- a/moses/src/Sentence.cpp +++ b/moses/src/Sentence.cpp @@ -27,6 +27,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "TranslationOptionCollectionText.h" #include "StaticData.h" #include "Util.h" +#include <boost/algorithm/string.hpp> using namespace std; @@ -92,6 +93,25 @@ int Sentence::Read(std::istream& in,const std::vector<FactorType>& factorOrder) if (meta.find("id") != meta.end()) { this->SetTranslationId(atol(meta["id"].c_str())); } + if (meta.find("docid") != meta.end()) { + this->SetDocumentId(atol(meta["docid"].c_str())); + this->SetUseTopicId(false); + this->SetUseTopicIdAndProb(false); + } + if (meta.find("topic") != meta.end()) { + vector<string> topic_params; + boost::split(topic_params, meta["topic"], boost::is_any_of("\t ")); + if (topic_params.size() == 1) { + this->SetTopicId(atol(topic_params[0].c_str())); + this->SetUseTopicId(true); + this->SetUseTopicIdAndProb(false); + } + else { + this->SetTopicIdAndProb(topic_params); + this->SetUseTopicId(false); + this->SetUseTopicIdAndProb(true); + } + } // parse XML markup in translation line //const StaticData &staticData = StaticData::Instance(); @@ -180,7 +200,7 @@ Sentence::CreateTranslationOptionCollection(const TranslationSystem* system) con } void Sentence::Print(std::ostream& out) const { - out<<*static_cast<Phrase const*>(this)<<"\n"; + out<<*static_cast<Phrase const*>(this); } diff --git a/moses/src/SourceWordDeletionFeature.cpp b/moses/src/SourceWordDeletionFeature.cpp new file mode 100644 index 000000000..52da2fdf7 --- /dev/null +++ b/moses/src/SourceWordDeletionFeature.cpp @@ -0,0 +1,94 @@ +#include <sstream> +#include "SourceWordDeletionFeature.h" +#include "Phrase.h" +#include "TargetPhrase.h" +#include "Hypothesis.h" +#include "ChartHypothesis.h" +#include "ScoreComponentCollection.h" +#include "TranslationOption.h" + +namespace Moses { + +using namespace std; + +bool SourceWordDeletionFeature::Load(const std::string &filePath) +{ + ifstream inFile(filePath.c_str()); + if (!inFile) + { + cerr << "could not open file " << filePath << endl; + return false; + } + + std::string line; + while (getline(inFile, line)) { + m_vocab.insert(line); + } + + inFile.close(); + + m_unrestricted = false; + return true; +} + +void SourceWordDeletionFeature::Evaluate( + const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const +{ + const TargetPhrase& targetPhrase = context.GetTargetPhrase(); + const AlignmentInfo &alignmentInfo = targetPhrase.GetAlignmentInfo(); + const AlignmentInfo::CollType &alignment = alignmentInfo.GetAlignments(); + ComputeFeatures(targetPhrase, accumulator, alignment); +} + +void SourceWordDeletionFeature::EvaluateChart( + const ChartBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const +{ + const AlignmentInfo &alignmentInfo = context.GetTargetPhrase().GetAlignmentInfo(); + const AlignmentInfo::CollType &alignment = alignmentInfo.GetTerminalAlignments(); + ComputeFeatures(context.GetTargetPhrase(), accumulator, alignment); +} + +void SourceWordDeletionFeature::ComputeFeatures(const TargetPhrase& targetPhrase, + ScoreComponentCollection* accumulator, + const AlignmentInfo::CollType &alignment) const +{ + // handle special case: unknown words (they have no word alignment) + size_t targetLength = targetPhrase.GetSize(); + size_t sourceLength = targetPhrase.GetSourcePhrase().GetSize(); + if (targetLength == 1 && sourceLength == 1) { + const Factor* f1 = targetPhrase.GetWord(0).GetFactor(1); + if (f1 && f1->GetString().compare(UNKNOWN_FACTOR) == 0) { + return; + } + } + + // flag aligned words + bool aligned[16]; + CHECK(sourceLength < 16); + for(size_t i=0; i<sourceLength; i++) + aligned[i] = false; + for (AlignmentInfo::const_iterator alignmentPoint = alignment.begin(); alignmentPoint != alignment.end(); alignmentPoint++) + aligned[ alignmentPoint->first ] = true; + + // process unaligned source words + for(size_t i=0; i<sourceLength; i++) { + if (!aligned[i]) { + Word w = targetPhrase.GetSourcePhrase().GetWord(i); + if (!w.IsNonTerminal()) { + const string &word = w.GetFactor(m_factorType)->GetString(); + if (word != "<s>" && word != "</s>") { + if (!m_unrestricted && m_vocab.find( word ) == m_vocab.end()) { + accumulator->PlusEquals(this,"OTHER",1); + } + else { + accumulator->PlusEquals(this,word,1); + } + } + } + } + } +} + +} diff --git a/moses/src/SourceWordDeletionFeature.h b/moses/src/SourceWordDeletionFeature.h new file mode 100644 index 000000000..c3a7b3f6b --- /dev/null +++ b/moses/src/SourceWordDeletionFeature.h @@ -0,0 +1,49 @@ +#ifndef moses_SourceWordDeletionFeature_h +#define moses_SourceWordDeletionFeature_h + +#include <string> +#include <map> + +#include "FeatureFunction.h" +#include "FactorCollection.h" +#include "AlignmentInfo.h" + +namespace Moses +{ + +/** Sets the features for source word deletion + */ +class SourceWordDeletionFeature : public StatelessFeatureFunction { +private: + std::set<std::string> m_vocab; + FactorType m_factorType; + bool m_unrestricted; + +public: + SourceWordDeletionFeature(FactorType factorType = 0): + StatelessFeatureFunction("swd", ScoreProducer::unlimited), + m_factorType(factorType), + m_unrestricted(true) + { + std::cerr << "Initializing source word deletion feature.." << std::endl; + } + + bool Load(const std::string &filePath); + void Evaluate(const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const; + + void EvaluateChart(const ChartBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const; + + void ComputeFeatures(const TargetPhrase& targetPhrase, + ScoreComponentCollection* accumulator, + const AlignmentInfo::CollType &alignment) const; + + // basic properties + std::string GetScoreProducerWeightShortName(unsigned) const { return "swd"; } + size_t GetNumInputScores() const { return 0; } +}; + +} + +#endif // moses_SourceWordDeletionFeature_h diff --git a/moses/src/SparsePhraseDictionaryFeature.cpp b/moses/src/SparsePhraseDictionaryFeature.cpp new file mode 100644 index 000000000..7177159df --- /dev/null +++ b/moses/src/SparsePhraseDictionaryFeature.cpp @@ -0,0 +1,17 @@ +#include "SparsePhraseDictionaryFeature.h" + + +namespace Moses +{ + + +void SparsePhraseDictionaryFeature::Evaluate( + const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const +{ + //not used +} + + + +} diff --git a/moses/src/SparsePhraseDictionaryFeature.h b/moses/src/SparsePhraseDictionaryFeature.h new file mode 100644 index 000000000..4185cac0d --- /dev/null +++ b/moses/src/SparsePhraseDictionaryFeature.h @@ -0,0 +1,40 @@ +#ifndef moses_SparsePhraseFeature_h +#define moses_SparsePhraseFeature_h + +#include <stdexcept> + +#include "FactorCollection.h" +#include "FeatureFunction.h" + +namespace Moses +{ + +/** + * Collection of sparse features attached to each phrase pair. + **/ +class SparsePhraseDictionaryFeature : public StatelessFeatureFunction { + +public: + SparsePhraseDictionaryFeature(): + StatelessFeatureFunction("stm", ScoreProducer::unlimited) {} + + void Evaluate(const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const; + + void EvaluateChart( + const ChartBasedFeatureContext& context, + ScoreComponentCollection*) const { + throw std::logic_error("SparsePhraseDictionaryFeature not valid in chart decoder"); + } + + // basic properties + std::string GetScoreProducerWeightShortName(unsigned) const { return "stm"; } + size_t GetNumInputScores() const { return 0; } + +}; + + +} + + +#endif diff --git a/moses/src/StaticData.cpp b/moses/src/StaticData.cpp index 48eef17b4..36e8757c7 100644 --- a/moses/src/StaticData.cpp +++ b/moses/src/StaticData.cpp @@ -34,12 +34,24 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "LM/Factory.h" #include "LexicalReordering.h" #include "GlobalLexicalModel.h" +#include "GlobalLexicalModelUnlimited.h" #include "SentenceStats.h" +#include "PhraseBoundaryFeature.h" #include "PhraseDictionary.h" +#include "SparsePhraseDictionaryFeature.h" +#include "PhrasePairFeature.h" +#include "PhraseLengthFeature.h" +#include "TargetWordInsertionFeature.h" +#include "SourceWordDeletionFeature.h" +#include "WordTranslationFeature.h" #include "UserMessage.h" #include "TranslationOption.h" +#include "TargetBigramFeature.h" +#include "TargetNgramFeature.h" #include "DecodeGraph.h" #include "InputFileStream.h" +#include "BleuScoreFeature.h" +#include "ScoreComponentCollection.h" #ifdef HAVE_SYNLM #include "SyntacticLanguageModel.h" @@ -74,16 +86,23 @@ static size_t CalcMax(size_t x, const vector<size_t>& y, const vector<size_t>& z StaticData StaticData::s_instance; StaticData::StaticData() - :m_numLinkParams(1) + :m_targetBigramFeature(NULL) + ,m_phraseBoundaryFeature(NULL) + ,m_phraseLengthFeature(NULL) + ,m_targetWordInsertionFeature(NULL) + ,m_sourceWordDeletionFeature(NULL) + ,m_numLinkParams(1) ,m_fLMsLoaded(false) ,m_sourceStartPosMattersForRecombination(false) ,m_inputType(SentenceInput) ,m_numInputScores(0) + ,m_bleuScoreFeature(NULL) ,m_detailedTranslationReportingFilePath() ,m_onlyDistinctNBest(false) ,m_factorDelimiter("|") // default delimiter between factors ,m_lmEnableOOVFeature(false) ,m_isAlwaysCreateDirectTranslationOption(false) + { m_maxFactorIdx[0] = 0; // source side m_maxFactorIdx[1] = 0; // target side @@ -190,6 +209,9 @@ bool StaticData::LoadData(Parameter *parameter) } else { m_nBestFactor = 20; } + + // explicit setting of distinct nbest + SetBooleanParameter( &m_onlyDistinctNBest, "distinct-nbest", false); //lattice samples if (m_parameter->GetParam("lattice-samples").size() ==2 ) { @@ -264,7 +286,7 @@ bool StaticData::LoadData(Parameter *parameter) // print all factors of output translations SetBooleanParameter( &m_reportAllFactorsNBest, "report-all-factors-in-n-best", false ); - // + // caching of translation options if (m_inputType == SentenceInput) { SetBooleanParameter( &m_useTransOptCache, "use-persistent-cache", true ); m_transOptCacheMaxSize = (m_parameter->GetParam("persistent-cache-size").size() > 0) @@ -273,6 +295,8 @@ bool StaticData::LoadData(Parameter *parameter) m_useTransOptCache = false; } + std::cerr << "transOptCache: " << m_useTransOptCache << std::endl; + std::cerr << "transOptCache max size: " << m_transOptCacheMaxSize << std::endl; //input factors const vector<string> &inputFactorVector = m_parameter->GetParam("input-factors"); @@ -317,14 +341,13 @@ bool StaticData::LoadData(Parameter *parameter) // word penalties for (size_t i = 0; i < m_parameter->GetParam("weight-w").size(); ++i) { float weightWordPenalty = Scan<float>( m_parameter->GetParam("weight-w")[i] ); - m_wordPenaltyProducers.push_back(new WordPenaltyProducer(m_scoreIndexManager)); - m_allWeights.push_back(weightWordPenalty); + m_wordPenaltyProducers.push_back(new WordPenaltyProducer()); + SetWeight(m_wordPenaltyProducers.back(), weightWordPenalty); } - float weightUnknownWord = (m_parameter->GetParam("weight-u").size() > 0) ? Scan<float>(m_parameter->GetParam("weight-u")[0]) : 1; - m_unknownWordPenaltyProducer = new UnknownWordPenaltyProducer(m_scoreIndexManager); - m_allWeights.push_back(weightUnknownWord); + m_unknownWordPenaltyProducer = new UnknownWordPenaltyProducer(); + SetWeight(m_unknownWordPenaltyProducer, weightUnknownWord); // reordering constraints m_maxDistortion = (m_parameter->GetParam("distortion-limit").size() > 0) ? @@ -335,6 +358,7 @@ bool StaticData::LoadData(Parameter *parameter) // settings for pruning m_maxHypoStackSize = (m_parameter->GetParam("stack").size() > 0) ? Scan<size_t>(m_parameter->GetParam("stack")[0]) : DEFAULT_MAX_HYPOSTACK_SIZE; + std::cerr << "max stack size: " << m_maxHypoStackSize << std::endl; m_minHypoStackDiversity = 0; if (m_parameter->GetParam("stack-diversity").size() > 0) { if (m_maxDistortion > 15) { @@ -396,6 +420,9 @@ bool StaticData::LoadData(Parameter *parameter) cerr << "Errror: Cannot use both n-best mbr and lattice mbr together" << endl; exit(1); } + + //mira training + SetBooleanParameter( &m_mira, "mira", false ); if (m_useLatticeMBR) m_mbr = true; @@ -529,8 +556,50 @@ bool StaticData::LoadData(Parameter *parameter) if (!LoadGenerationTables()) return false; if (!LoadPhraseTables()) return false; if (!LoadGlobalLexicalModel()) return false; + if (!LoadGlobalLexicalModelUnlimited()) return false; if (!LoadDecodeGraphs()) return false; - + if (!LoadReferences()) return false; + if (!LoadDiscrimLMFeature()) return false; + if (!LoadPhrasePairFeature()) return false; + if (!LoadPhraseBoundaryFeature()) return false; + if (!LoadPhraseLengthFeature()) return false; + if (!LoadTargetWordInsertionFeature()) return false; + if (!LoadSourceWordDeletionFeature()) return false; + if (!LoadWordTranslationFeature()) return false; + + // report individual sparse features in n-best list + if (m_parameter->GetParam("report-sparse-features").size() > 0) { + for(size_t i=0; i<m_parameter->GetParam("report-sparse-features").size(); i++) { + const std::string &name = m_parameter->GetParam("report-sparse-features")[i]; + if (m_targetBigramFeature && name.compare(m_targetBigramFeature->GetScoreProducerWeightShortName(0)) == 0) + m_targetBigramFeature->SetSparseFeatureReporting(); + if (m_targetNgramFeatures.size() > 0) + for (size_t i=0; i < m_targetNgramFeatures.size(); ++i) + if (name.compare(m_targetNgramFeatures[i]->GetScoreProducerWeightShortName(0)) == 0) + m_targetNgramFeatures[i]->SetSparseFeatureReporting(); + if (m_phraseBoundaryFeature && name.compare(m_phraseBoundaryFeature->GetScoreProducerWeightShortName(0)) == 0) + m_phraseBoundaryFeature->SetSparseFeatureReporting(); + if (m_phraseLengthFeature && name.compare(m_phraseLengthFeature->GetScoreProducerWeightShortName(0)) == 0) + m_phraseLengthFeature->SetSparseFeatureReporting(); + if (m_targetWordInsertionFeature && name.compare(m_targetWordInsertionFeature->GetScoreProducerWeightShortName(0)) == 0) + m_targetWordInsertionFeature->SetSparseFeatureReporting(); + if (m_sourceWordDeletionFeature && name.compare(m_sourceWordDeletionFeature->GetScoreProducerWeightShortName(0)) == 0) + m_sourceWordDeletionFeature->SetSparseFeatureReporting(); + if (m_wordTranslationFeatures.size() > 0) + for (size_t i=0; i < m_wordTranslationFeatures.size(); ++i) + if (name.compare(m_wordTranslationFeatures[i]->GetScoreProducerWeightShortName(0)) == 0) + m_wordTranslationFeatures[i]->SetSparseFeatureReporting(); + if (m_phrasePairFeatures.size() > 0) + for (size_t i=0; i < m_phrasePairFeatures.size(); ++i) + if (name.compare(m_phrasePairFeatures[i]->GetScoreProducerWeightShortName(0)) == 0) + m_wordTranslationFeatures[i]->SetSparseFeatureReporting(); + for (size_t j = 0; j < m_sparsePhraseDictionary.size(); ++j) { + if (m_sparsePhraseDictionary[j] && name.compare(m_sparsePhraseDictionary[j]->GetScoreProducerWeightShortName(0)) == 0) { + m_sparsePhraseDictionary[j]->SetSparseFeatureReporting(); + } + } + } + } //configure the translation systems with these tables vector<string> tsConfig = m_parameter->GetParam("translation-systems"); @@ -554,6 +623,7 @@ bool StaticData::LoadData(Parameter *parameter) } } + TranslationSystem* tmpTS; for (size_t i = 0; i < tsConfig.size(); ++i) { vector<string> config = Tokenize(tsConfig[i]); if (config.size() % 2 != 1) { @@ -561,6 +631,7 @@ bool StaticData::LoadData(Parameter *parameter) } m_translationSystems.insert(pair<string, TranslationSystem>(config[0], TranslationSystem(config[0],m_wordPenaltyProducers[i],m_unknownWordPenaltyProducer,m_distortionScoreProducers[i]))); + tmpTS = &(m_translationSystems.find(config[0])->second); for (size_t j = 1; j < config.size(); j += 2) { const string& id = config[j]; const string& tables = config[j+1]; @@ -609,18 +680,119 @@ bool StaticData::LoadData(Parameter *parameter) //Instigate dictionary loading m_translationSystems.find(config[0])->second.ConfigDictionaries(); - - //Add any other features here. + if (m_bleuScoreFeature) { + m_translationSystems.find(config[0])->second.AddFeatureFunction(m_bleuScoreFeature); + } + if (m_targetBigramFeature) { + m_translationSystems.find(config[0])->second.AddFeatureFunction(m_targetBigramFeature); + } + if (m_targetNgramFeatures.size() > 0) { + for (size_t i=0; i < m_targetNgramFeatures.size(); ++i) + m_translationSystems.find(config[0])->second.AddFeatureFunction(m_targetNgramFeatures[i]); + } + if (m_phraseBoundaryFeature) { + m_translationSystems.find(config[0])->second.AddFeatureFunction(m_phraseBoundaryFeature); + } + if (m_phraseLengthFeature) { + m_translationSystems.find(config[0])->second.AddFeatureFunction(m_phraseLengthFeature); + } + if (m_targetWordInsertionFeature) { + m_translationSystems.find(config[0])->second.AddFeatureFunction(m_targetWordInsertionFeature); + } + if (m_sourceWordDeletionFeature) { + m_translationSystems.find(config[0])->second.AddFeatureFunction(m_sourceWordDeletionFeature); + } + if (m_wordTranslationFeatures.size() > 0) { + for (size_t i=0; i < m_wordTranslationFeatures.size(); ++i) + m_translationSystems.find(config[0])->second.AddFeatureFunction(m_wordTranslationFeatures[i]); + } + if (m_phrasePairFeatures.size() > 0) { + for (size_t i=0; i < m_phrasePairFeatures.size(); ++i) + m_translationSystems.find(config[0])->second.AddFeatureFunction(m_phrasePairFeatures[i]); + } #ifdef HAVE_SYNLM if (m_syntacticLanguageModel != NULL) { m_translationSystems.find(config[0])->second.AddFeatureFunction(m_syntacticLanguageModel); } #endif + for (size_t i = 0; i < m_sparsePhraseDictionary.size(); ++i) { + if (m_sparsePhraseDictionary[i]) { + m_translationSystems.find(config[0])->second.AddFeatureFunction(m_sparsePhraseDictionary[i]); + } + } + if (m_globalLexicalModelsUnlimited.size() > 0) { + for (size_t i=0; i < m_globalLexicalModelsUnlimited.size(); ++i) + m_translationSystems.find(config[0])->second.AddFeatureFunction(m_globalLexicalModelsUnlimited[i]); + } } + //Load extra feature weights + //NB: These are common to all translation systems (at the moment!) + vector<string> extraWeightConfig = m_parameter->GetParam("weight-file"); + if (extraWeightConfig.size()) { + if (extraWeightConfig.size() != 1) { + UserMessage::Add("One argument should be supplied for weight-file"); + return false; + } + ScoreComponentCollection extraWeights; + if (!extraWeights.Load(extraWeightConfig[0])) { + UserMessage::Add("Unable to load weights from " + extraWeightConfig[0]); + return false; + } + + + // DLM: apply additional weight to sparse features if applicable + for (size_t i = 0; i < m_targetNgramFeatures.size(); ++i) { + float weight = m_targetNgramFeatures[i]->GetSparseProducerWeight(); + if (weight != 1) { + tmpTS->AddSparseProducer(m_targetNgramFeatures[i]); + cerr << "dlm sparse producer weight: " << weight << endl; + } + } - m_scoreIndexManager.InitFeatureNames(); + // GLM: apply additional weight to sparse features if applicable + for (size_t i = 0; i < m_globalLexicalModelsUnlimited.size(); ++i) { + float weight = m_globalLexicalModelsUnlimited[i]->GetSparseProducerWeight(); + if (weight != 1) { + tmpTS->AddSparseProducer(m_globalLexicalModelsUnlimited[i]); + cerr << "glm sparse producer weight: " << weight << endl; + } + } + + // WT: apply additional weight to sparse features if applicable + for (size_t i = 0; i < m_wordTranslationFeatures.size(); ++i) { + float weight = m_wordTranslationFeatures[i]->GetSparseProducerWeight(); + if (weight != 1) { + tmpTS->AddSparseProducer(m_wordTranslationFeatures[i]); + cerr << "wt sparse producer weight: " << weight << endl; + if (m_mira) + m_metaFeatureProducer = new MetaFeatureProducer("wt"); + } + } + + // PP: apply additional weight to sparse features if applicable + for (size_t i = 0; i < m_phrasePairFeatures.size(); ++i) { + float weight = m_phrasePairFeatures[i]->GetSparseProducerWeight(); + if (weight != 1) { + tmpTS->AddSparseProducer(m_phrasePairFeatures[i]); + cerr << "pp sparse producer weight: " << weight << endl; + if (m_mira) + m_metaFeatureProducer = new MetaFeatureProducer("pp"); + } + } + + // PB: apply additional weight to sparse features if applicable + if (m_phraseBoundaryFeature) { + float weight = m_phraseBoundaryFeature->GetSparseProducerWeight(); + if (weight != 1) { + tmpTS->AddSparseProducer(m_phraseBoundaryFeature); + cerr << "pb sparse producer weight: " << weight << endl; + } + } + + m_allWeights.PlusEquals(extraWeights); + } return true; } @@ -644,8 +816,21 @@ void StaticData::SetBooleanParameter( bool *parameter, string parameterName, boo } } +void StaticData::SetWeight(const ScoreProducer* sp, float weight) +{ + m_allWeights.Resize(); + m_allWeights.Assign(sp,weight); +} + +void StaticData::SetWeights(const ScoreProducer* sp, const std::vector<float>& weights) +{ + m_allWeights.Resize(); + m_allWeights.Assign(sp,weights); +} + StaticData::~StaticData() { + RemoveAllInColl(m_sparsePhraseDictionary); RemoveAllInColl(m_phraseDictionary); RemoveAllInColl(m_generationDictionary); RemoveAllInColl(m_reorderModels); @@ -666,8 +851,19 @@ StaticData::~StaticData() // small score producers delete m_unknownWordPenaltyProducer; - - //delete m_parameter; + delete m_targetBigramFeature; + for (size_t i=0; i < m_targetNgramFeatures.size(); ++i) + delete m_targetNgramFeatures[i]; + delete m_phraseBoundaryFeature; + delete m_phraseLengthFeature; + delete m_targetWordInsertionFeature; + delete m_sourceWordDeletionFeature; + for (size_t i=0; i < m_wordTranslationFeatures.size(); ++i) + delete m_wordTranslationFeatures[i]; + for (size_t i=0; i < m_phrasePairFeatures.size(); ++i) + delete m_phrasePairFeatures[i]; + for (size_t i=0; i < m_globalLexicalModelsUnlimited.size(); ++i) + delete m_globalLexicalModelsUnlimited[i]; // memory pools Phrase::FinalizeMemPool(); @@ -796,7 +992,7 @@ bool StaticData::LoadLexicalReorderingModel() string filePath = spec[3]; - m_reorderModels.push_back(new LexicalReordering(input, output, modelType, filePath, mweights)); + m_reorderModels.push_back(new LexicalReordering(input, output, LexicalReorderingConfiguration(modelType), filePath, mweights)); } return true; } @@ -825,7 +1021,73 @@ bool StaticData::LoadGlobalLexicalModel() } vector<FactorType> inputFactors = Tokenize<FactorType>(factors[0],","); vector<FactorType> outputFactors = Tokenize<FactorType>(factors[1],","); - m_globalLexicalModels.push_back( new GlobalLexicalModel( spec[1], weight[i], inputFactors, outputFactors ) ); + m_globalLexicalModels.push_back( new GlobalLexicalModel( spec[1], inputFactors, outputFactors ) ); + SetWeight(m_globalLexicalModels.back(),weight[i]); + } + return true; +} + +bool StaticData::LoadGlobalLexicalModelUnlimited() +{ + const vector<float> &weight = Scan<float>(m_parameter->GetParam("weight-glm")); + const vector<string> &modelSpec = m_parameter->GetParam("glm-feature"); + + if (weight.size() != 0 && weight.size() != modelSpec.size()) { + std::cerr << "number of sparse producer weights and model specs for the global lexical model unlimited " + "does not match (" << weight.size() << " != " << modelSpec.size() << ")" << std::endl; + return false; + } + + for (size_t i = 0; i < modelSpec.size(); i++ ) { + bool ignorePunctuation = true, biasFeature = false, restricted = false; + size_t context = 0; + string filenameSource, filenameTarget; + vector< string > factors; + vector< string > spec = Tokenize(modelSpec[i]," "); + + // read optional punctuation and bias specifications + if (spec.size() > 0) { + if (spec.size() != 2 && spec.size() != 3 && spec.size() != 4 && spec.size() != 6) { + UserMessage::Add("Format of glm feature is <factor-src>-<factor-tgt> [ignore-punct] [use-bias] " + "[context-type] [filename-src filename-tgt]"); + return false; + } + + factors = Tokenize(spec[0],"-"); + if (spec.size() >= 2) + ignorePunctuation = Scan<size_t>(spec[1]); + if (spec.size() >= 3) + biasFeature = Scan<size_t>(spec[2]); + if (spec.size() >= 4) + context = Scan<size_t>(spec[3]); + if (spec.size() == 6) { + filenameSource = spec[4]; + filenameTarget = spec[5]; + restricted = true; + } + } + else + factors = Tokenize(modelSpec[i],"-"); + + if ( factors.size() != 2 ) { + UserMessage::Add("Wrong factor definition for global lexical model unlimited: " + modelSpec[i]); + return false; + } + + const vector<FactorType> inputFactors = Tokenize<FactorType>(factors[0],","); + const vector<FactorType> outputFactors = Tokenize<FactorType>(factors[1],","); + throw runtime_error("GlobalLexicalModelUnlimited should be reimplemented as a stateful feature"); + GlobalLexicalModelUnlimited* glmu = NULL; // new GlobalLexicalModelUnlimited(inputFactors, outputFactors, biasFeature, ignorePunctuation, context); + m_globalLexicalModelsUnlimited.push_back(glmu); + if (restricted) { + cerr << "loading word translation word lists from " << filenameSource << " and " << filenameTarget << endl; + if (!glmu->Load(filenameSource, filenameTarget)) { + UserMessage::Add("Unable to load word lists for word translation feature from files " + filenameSource + " and " + filenameTarget); + return false; + } + } + if (weight.size() > i) + m_globalLexicalModelsUnlimited[i]->SetSparseProducerWeight(weight[i]); } return true; } @@ -836,10 +1098,6 @@ bool StaticData::LoadLanguageModels() // weights vector<float> weightAll = Scan<float>(m_parameter->GetParam("weight-l")); - for (size_t i = 0 ; i < weightAll.size() ; i++) { - m_allWeights.push_back(weightAll[i]); - } - // dictionary upper-bounds fo all IRST LMs vector<int> LMdub = Scan<int>(m_parameter->GetParam("lmodel-dub")); if (m_parameter->GetParam("lmodel-dub").size() == 0) { @@ -855,7 +1113,7 @@ bool StaticData::LoadLanguageModels() for(size_t i=0; i<lmVector.size(); i++) { LanguageModel* lm = NULL; if (languageModelsLoaded.find(lmVector[i]) != languageModelsLoaded.end()) { - lm = languageModelsLoaded[lmVector[i]]->Duplicate(m_scoreIndexManager); + lm = languageModelsLoaded[lmVector[i]]->Duplicate(); } else { vector<string> token = Tokenize(lmVector[i]); if (token.size() != 4 && token.size() != 5 ) { @@ -888,7 +1146,6 @@ bool StaticData::LoadLanguageModels() , factorTypes , nGramOrder , languageModelFile - , m_scoreIndexManager , LMdub[i]); if (lm == NULL) { UserMessage::Add("no LM created. We probably don't have it compiled"); @@ -898,6 +1155,14 @@ bool StaticData::LoadLanguageModels() } m_languageModel.Add(lm); + if (m_lmEnableOOVFeature) { + vector<float> weights(2); + weights[0] = weightAll.at(i*2); + weights[1] = weightAll.at(i*2+1); + SetWeights(lm,weights); + } else { + SetWeight(lm,weightAll[i]); + } } } // flag indicating that language models were loaded, @@ -940,16 +1205,18 @@ bool StaticData::LoadGenerationTables() VERBOSE(1, filePath << endl); - m_generationDictionary.push_back(new GenerationDictionary(numFeatures, m_scoreIndexManager, input,output)); + m_generationDictionary.push_back(new GenerationDictionary(numFeatures, input,output)); CHECK(m_generationDictionary.back() && "could not create GenerationDictionary"); if (!m_generationDictionary.back()->Load(filePath, Output)) { delete m_generationDictionary.back(); return false; } + vector<float> gdWeights; for(size_t i = 0; i < numFeatures; i++) { CHECK(currWeightNum < weight.size()); - m_allWeights.push_back(weight[currWeightNum++]); + gdWeights.push_back(weight[currWeightNum++]); } + SetWeights(m_generationDictionary.back(), gdWeights); } if (currWeightNum != weight.size()) { TRACE_ERR( " [WARNING] config file has " << weight.size() << " generation weights listed, but the configuration for generation files indicates there should be " << currWeightNum << "!\n"); @@ -1069,6 +1336,7 @@ bool StaticData::LoadPhraseTables() for (size_t currScore = 0 ; currScore < numScoreComponent; currScore++) weight.push_back(weightAll[weightAllOffset + currScore]); + if(weight.size() - tableInputScores != numScoreComponent) { stringstream strme; strme << "Your phrase table has " << numScoreComponent @@ -1088,7 +1356,6 @@ bool StaticData::LoadPhraseTables() CHECK(numScoreComponent==weight.size()); - std::copy(weight.begin(),weight.end(),std::back_inserter(m_allWeights)); //This is needed for regression testing, but the phrase table //might not really be loading here @@ -1096,19 +1363,30 @@ bool StaticData::LoadPhraseTables() PrintUserTime(string("Start loading PhraseTable ") + filePath); VERBOSE(1,"filePath: " << filePath <<endl); + //optional create sparse phrase feature + SparsePhraseDictionaryFeature* spdf = NULL; + if (token.size() >= 6 && token[5] == "sparse") { + spdf = new SparsePhraseDictionaryFeature(); + } + m_sparsePhraseDictionary.push_back(spdf); + + PhraseDictionaryFeature* pdf = new PhraseDictionaryFeature( implementation + , spdf , numScoreComponent , (currDict==0 ? m_numInputScores : 0) , input , output , filePath , weight + , currDict , maxTargetPhrase[index] , targetPath, alignmentsFile); m_phraseDictionary.push_back(pdf); + SetWeights(m_phraseDictionary.back(),weight); @@ -1189,8 +1467,8 @@ void StaticData::LoadPhraseBasedParameters() } for (size_t i = 0; i < distortionWeightCount; ++i) { float weightDistortion = Scan<float>(distortionWeights[i]); - m_distortionScoreProducers.push_back(new DistortionScoreProducer(m_scoreIndexManager)); - m_allWeights.push_back(weightDistortion); + m_distortionScoreProducers.push_back(new DistortionScoreProducer()); + SetWeight(m_distortionScoreProducers.back(), weightDistortion); } } @@ -1257,6 +1535,7 @@ bool StaticData::LoadDecodeGraphs() DecodeGraph *decodeGraph; if (m_searchAlgorithm == ChartDecoding) { size_t maxChartSpan = (decodeGraphInd < maxChartSpans.size()) ? maxChartSpans[decodeGraphInd] : DEFAULT_MAX_CHART_SPAN; + cerr << "max-chart-span: " << maxChartSpans[decodeGraphInd] << endl; decodeGraph = new DecodeGraph(m_decodeGraphs.size(), maxChartSpan); } else { decodeGraph = new DecodeGraph(m_decodeGraphs.size()); @@ -1284,18 +1563,387 @@ bool StaticData::LoadDecodeGraphs() return true; } +bool StaticData::LoadReferences() +{ + vector<string> bleuWeightStr = m_parameter->GetParam("weight-bl"); + vector<string> referenceFiles = m_parameter->GetParam("references"); + if ((!referenceFiles.size() && bleuWeightStr.size()) || (referenceFiles.size() && !bleuWeightStr.size())) { + UserMessage::Add("You cannot use the bleu feature without references, and vice-versa"); + return false; + } + if (!referenceFiles.size()) { + return true; + } + if (bleuWeightStr.size() > 1) { + UserMessage::Add("Can only specify one weight for the bleu feature"); + return false; + } + + float bleuWeight = Scan<float>(bleuWeightStr[0]); + m_bleuScoreFeature = new BleuScoreFeature(); + SetWeight(m_bleuScoreFeature, bleuWeight); + + cerr << "Loading reference file " << referenceFiles[0] << endl; + vector<vector<string> > references(referenceFiles.size()); + for (size_t i =0; i < referenceFiles.size(); ++i) { + ifstream in(referenceFiles[i].c_str()); + if (!in) { + stringstream strme; + strme << "Unable to load references from " << referenceFiles[i]; + UserMessage::Add(strme.str()); + return false; + } + string line; + while (getline(in,line)) { +/* if (GetSearchAlgorithm() == ChartDecoding) { + stringstream tmp; + tmp << "<s> " << line << " </s>"; + line = tmp.str(); + }*/ + references[i].push_back(line); + } + if (i > 0) { + if (references[i].size() != references[i-1].size()) { + UserMessage::Add("Reference files are of different lengths"); + return false; + } + } + in.close(); + } + //Set the references in the bleu feature + m_bleuScoreFeature->LoadReferences(references); + return true; +} + +bool StaticData::LoadDiscrimLMFeature() +{ + // only load if specified + const vector<string> &wordFile = m_parameter->GetParam("dlm-model"); + if (wordFile.empty()) { + return true; + } + cerr << "Loading " << wordFile.size() << " discriminative language model(s).." << endl; + + // if this weight is specified, the sparse DLM weights will be scaled with an additional weight + vector<string> dlmWeightStr = m_parameter->GetParam("weight-dlm"); + vector<float> dlmWeights; + for (size_t i=0; i<dlmWeightStr.size(); ++i) + dlmWeights.push_back(Scan<float>(dlmWeightStr[i])); + + for (size_t i = 0; i < wordFile.size(); ++i) { + vector<string> tokens = Tokenize(wordFile[i]); + if (tokens.size() != 4) { + UserMessage::Add("Format of discriminative language model parameter is <order> <factor> <include-lower-ngrams> <filename>"); + return false; + } + + size_t order = Scan<size_t>(tokens[0]); + FactorType factorId = Scan<size_t>(tokens[1]); + bool include_lower_ngrams = Scan<bool>(tokens[2]); + string filename = tokens[3]; + + if (order == 2 && !include_lower_ngrams) { // TODO: remove TargetBigramFeature ? + m_targetBigramFeature = new TargetBigramFeature(factorId); + cerr << "loading vocab from " << filename << endl; + if (!m_targetBigramFeature->Load(filename)) { + UserMessage::Add("Unable to load word list from file " + filename); + return false; + } + } + else { + if (m_searchAlgorithm == ChartDecoding && !include_lower_ngrams) { + UserMessage::Add("Excluding lower order DLM ngrams is currently not supported for chart decoding."); + return false; + } + + m_targetNgramFeatures.push_back(new TargetNgramFeature(factorId, order, include_lower_ngrams)); + if (i < dlmWeights.size()) + m_targetNgramFeatures[i]->SetSparseProducerWeight(dlmWeights[i]); + cerr << "loading vocab from " << filename << endl; + if (!m_targetNgramFeatures[i]->Load(filename)) { + UserMessage::Add("Unable to load word list from file " + filename); + return false; + } + } + } + + return true; +} -void StaticData::SetWeightsForScoreProducer(const ScoreProducer* sp, const std::vector<float>& weights) +bool StaticData::LoadPhraseBoundaryFeature() { - const size_t id = sp->GetScoreBookkeepingID(); - const size_t begin = m_scoreIndexManager.GetBeginIndex(id); - const size_t end = m_scoreIndexManager.GetEndIndex(id); - CHECK(end - begin == weights.size()); - if (m_allWeights.size() < end) - m_allWeights.resize(end); - std::vector<float>::const_iterator weightIter = weights.begin(); - for (size_t i = begin; i < end; i++) - m_allWeights[i] = *weightIter++; + const vector<float> &weight = Scan<float>(m_parameter->GetParam("weight-pb")); + if (weight.size() > 1) { + std::cerr << "only one sparse producer weight allowed for the phrase boundary feature" << std::endl; + return false; + } + + const vector<string> &phraseBoundarySourceFactors = + m_parameter->GetParam("phrase-boundary-source-feature"); + const vector<string> &phraseBoundaryTargetFactors = + m_parameter->GetParam("phrase-boundary-target-feature"); + if (phraseBoundarySourceFactors.size() == 0 && phraseBoundaryTargetFactors.size() == 0) { + return true; + } + if (phraseBoundarySourceFactors.size() > 1) { + UserMessage::Add("Need to specify comma separated list of source factors for phrase boundary"); + return false; + } + if (phraseBoundaryTargetFactors.size() > 1) { + UserMessage::Add("Need to specify comma separated list of target factors for phrase boundary"); + return false; + } + FactorList sourceFactors; + FactorList targetFactors; + if (phraseBoundarySourceFactors.size()) { + sourceFactors = Tokenize<FactorType>(phraseBoundarySourceFactors[0],","); + } + if (phraseBoundaryTargetFactors.size()) { + targetFactors = Tokenize<FactorType>(phraseBoundaryTargetFactors[0],","); + } + //cerr << "source "; for (size_t i = 0; i < sourceFactors.size(); ++i) cerr << sourceFactors[i] << " "; cerr << endl; + //cerr << "target "; for (size_t i = 0; i < targetFactors.size(); ++i) cerr << targetFactors[i] << " "; cerr << endl; + m_phraseBoundaryFeature = new PhraseBoundaryFeature(sourceFactors,targetFactors); + if (weight.size() > 0) + m_phraseBoundaryFeature->SetSparseProducerWeight(weight[0]); + return true; +} + +bool StaticData::LoadPhrasePairFeature() +{ + const vector<float> &weight = Scan<float>(m_parameter->GetParam("weight-pp")); + if (weight.size() > 1) { + std::cerr << "Only one sparse producer weight allowed for the phrase pair feature" << std::endl; + return false; + } + + const vector<string> ¶meters = m_parameter->GetParam("phrase-pair-feature"); + if (parameters.size() == 0) return true; + + for (size_t i=0; i<parameters.size(); ++i) { + vector<string> tokens = Tokenize(parameters[i]); + if (! (tokens.size() >= 1 && tokens.size() <= 6)) { + UserMessage::Add("Format for phrase pair feature: --phrase-pair-feature <factor-src>-<factor-tgt> " + "[simple source-trigger] [ignore-punctuation] [domain-trigger] [filename-src]"); + return false; + } + + vector <string> factors; + if (tokens.size() == 2) + factors = Tokenize(tokens[0]," "); + else + factors = Tokenize(tokens[0],"-"); + + size_t sourceFactorId = Scan<size_t>(factors[0]); + size_t targetFactorId = Scan<size_t>(factors[1]); + bool simple = true, sourceContext = false, ignorePunctuation = false, domainTrigger = false; + if (tokens.size() >= 3) { + simple = Scan<size_t>(tokens[1]); + sourceContext = Scan<size_t>(tokens[2]); + } + if (tokens.size() >= 4) + ignorePunctuation = Scan<size_t>(tokens[3]); + if (tokens.size() >= 5) + domainTrigger = Scan<size_t>(tokens[4]); + + m_phrasePairFeatures.push_back(new PhrasePairFeature(sourceFactorId, targetFactorId, simple, sourceContext, + ignorePunctuation, domainTrigger)); + if (weight.size() > i) + m_phrasePairFeatures[i]->SetSparseProducerWeight(weight[i]); + + // load word list + if (tokens.size() == 6) { + string filenameSource = tokens[5]; + if (domainTrigger) { + const vector<string> &texttype = m_parameter->GetParam("text-type"); + if (texttype.size() != 1) { + UserMessage::Add("Need texttype to load dictionary for domain triggers."); + return false; + } + stringstream filename(filenameSource + "." + texttype[0]); + filenameSource = filename.str(); + cerr << "loading word translation term list from " << filenameSource << endl; + } + else { + cerr << "loading word translation word list from " << filenameSource << endl; + } + if (!m_phrasePairFeatures[i]->Load(filenameSource)) { + UserMessage::Add("Unable to load word lists for word translation feature from files " + filenameSource); + return false; + } + } + } + return true; +} + +bool StaticData::LoadPhraseLengthFeature() +{ + if (m_parameter->isParamSpecified("phrase-length-feature")) { + m_phraseLengthFeature = new PhraseLengthFeature(); + } + return true; +} + +bool StaticData::LoadTargetWordInsertionFeature() +{ + const vector<string> ¶meters = m_parameter->GetParam("target-word-insertion-feature"); + if (parameters.empty()) + return true; + + if (parameters.size() != 1) { + UserMessage::Add("Can only have one target-word-insertion-feature"); + return false; + } + + vector<string> tokens = Tokenize(parameters[0]); + if (tokens.size() != 1 && tokens.size() != 2) { + UserMessage::Add("Format of target word insertion feature parameter is: --target-word-insertion-feature <factor> [filename]"); + return false; + } + + if (!m_UseAlignmentInfo && GetSearchAlgorithm() != ChartDecoding) { + UserMessage::Add("Target word insertion feature needs word alignments in phrase table."); + return false; + } + + // set factor + FactorType factorId = Scan<size_t>(tokens[0]); + m_targetWordInsertionFeature = new TargetWordInsertionFeature(factorId); + + // load word list for restricted feature set + if (tokens.size() == 2) { + string filename = tokens[1]; + cerr << "loading target word insertion word list from " << filename << endl; + if (!m_targetWordInsertionFeature->Load(filename)) { + UserMessage::Add("Unable to load word list for target word insertion feature from file " + filename); + return false; + } + } + + return true; +} + +bool StaticData::LoadSourceWordDeletionFeature() +{ + const vector<string> ¶meters = m_parameter->GetParam("source-word-deletion-feature"); + if (parameters.empty()) + return true; + + if (parameters.size() != 1) { + UserMessage::Add("Can only have one source-word-deletion-feature"); + return false; + } + + vector<string> tokens = Tokenize(parameters[0]); + if (tokens.size() != 1 && tokens.size() != 2) { + UserMessage::Add("Format of source word deletion feature parameter is: --source-word-deletion-feature <factor> [filename]"); + return false; + } + + if (!m_UseAlignmentInfo && GetSearchAlgorithm() != ChartDecoding) { + UserMessage::Add("Source word deletion feature needs word alignments in phrase table."); + return false; + } + + // set factor + FactorType factorId = Scan<size_t>(tokens[0]); + m_sourceWordDeletionFeature = new SourceWordDeletionFeature(factorId); + + // load word list for restricted feature set + if (tokens.size() == 2) { + string filename = tokens[1]; + cerr << "loading source word deletion word list from " << filename << endl; + if (!m_sourceWordDeletionFeature->Load(filename)) { + UserMessage::Add("Unable to load word list for source word deletion feature from file " + filename); + return false; + } + } + + return true; +} + +bool StaticData::LoadWordTranslationFeature() +{ + const vector<float> &weight = Scan<float>(m_parameter->GetParam("weight-wt")); + if (weight.size() > 1) { + std::cerr << "Only one sparse producer weight allowed for the word translation feature" << std::endl; + return false; + } + + const vector<string> ¶meters = m_parameter->GetParam("word-translation-feature"); + if (parameters.empty()) + return true; + + for (size_t i=0; i<parameters.size(); ++i) { + vector<string> tokens = Tokenize(parameters[i]); + if (tokens.size() != 1 && !(tokens.size() >= 4 && tokens.size() <= 8)) { + UserMessage::Add("Format of word translation feature parameter is: --word-translation-feature <factor-src>-<factor-tgt> " + "[simple source-trigger target-trigger] [ignore-punctuation] [domain-trigger] [filename-src] [filename-tgt]"); + return false; + } + + if (!m_UseAlignmentInfo && GetSearchAlgorithm() != ChartDecoding) { + UserMessage::Add("Word translation feature needs word alignments in phrase table."); + return false; + } + + // set factor + vector <string> factors = Tokenize(tokens[0],"-"); + FactorType factorIdSource = Scan<size_t>(factors[0]); + FactorType factorIdTarget = Scan<size_t>(factors[1]); + + bool simple = true, sourceTrigger = false, targetTrigger = false, ignorePunctuation = false, domainTrigger = false; + if (tokens.size() >= 4) { + simple = Scan<size_t>(tokens[1]); + sourceTrigger = Scan<size_t>(tokens[2]); + targetTrigger = Scan<size_t>(tokens[3]); + } + if (tokens.size() >= 5) { + ignorePunctuation = Scan<size_t>(tokens[4]); + } + + if (tokens.size() >= 6) { + domainTrigger = Scan<size_t>(tokens[5]); + } + + m_wordTranslationFeatures.push_back(new WordTranslationFeature(factorIdSource, factorIdTarget, simple, + sourceTrigger, targetTrigger, ignorePunctuation, domainTrigger)); + if (weight.size() > i) + m_wordTranslationFeatures[i]->SetSparseProducerWeight(weight[i]); + + // load word list for restricted feature set + if (tokens.size() == 7) { + string filenameSource = tokens[6]; + if (domainTrigger) { + const vector<string> &texttype = m_parameter->GetParam("text-type"); + if (texttype.size() != 1) { + UserMessage::Add("Need texttype to load dictionary for domain triggers."); + return false; + } + stringstream filename(filenameSource + "." + texttype[0]); + filenameSource = filename.str(); + cerr << "loading word translation term list from " << filenameSource << endl; + } + else { + cerr << "loading word translation word lists from " << filenameSource << endl; + } + if (!m_wordTranslationFeatures[i]->Load(filenameSource, "")) { + UserMessage::Add("Unable to load word lists for word translation feature from files " + filenameSource); + return false; + } + } + else if (tokens.size() == 8) { + string filenameSource = tokens[6]; + string filenameTarget = tokens[7]; + cerr << "loading word translation word lists from " << filenameSource << " and " << filenameTarget << endl; + if (!m_wordTranslationFeatures[i]->Load(filenameSource, filenameTarget)) { + UserMessage::Add("Unable to load word lists for word translation feature from files " + filenameSource + " and " + filenameTarget); + return false; + } + } + } + + return true; } const TranslationOptionList* StaticData::FindTransOptListInCache(const DecodeGraph &decodeGraph, const Phrase &sourcePhrase) const @@ -1305,7 +1953,7 @@ const TranslationOptionList* StaticData::FindTransOptListInCache(const DecodeGra boost::mutex::scoped_lock lock(m_transOptCacheMutex); #endif std::map<std::pair<size_t, Phrase>, std::pair<TranslationOptionList*,clock_t> >::iterator iter - = m_transOptCache.find(key); + = m_transOptCache.find(key); if (iter == m_transOptCache.end()) return NULL; iter->second.second = clock(); // update last used time @@ -1360,6 +2008,88 @@ void StaticData::ClearTransOptionCache() const { } } +void StaticData::ReLoadParameter() +{ + m_verboseLevel = 1; + if (m_parameter->GetParam("verbose").size() == 1) { + m_verboseLevel = Scan<size_t>( m_parameter->GetParam("verbose")[0]); + } + + // check whether "weight-u" is already set + if (m_parameter->isParamShortNameSpecified("u")) { + if (m_parameter->GetParamShortName("u").size() < 1 ) { + PARAM_VEC w(1,"1.0"); + m_parameter->OverwriteParamShortName("u", w); + } + } + + //loop over all ScoreProducer to update weights + const TranslationSystem &transSystem = GetTranslationSystem(TranslationSystem::DEFAULT); + + std::vector<const ScoreProducer*>::const_iterator iterSP; + for (iterSP = transSystem.GetFeatureFunctions().begin() ; iterSP != transSystem.GetFeatureFunctions().end() ; ++iterSP) { + std::string paramShortName = (*iterSP)->GetScoreProducerWeightShortName(); + vector<float> Weights = Scan<float>(m_parameter->GetParamShortName(paramShortName)); + + if (paramShortName == "d") { //basic distortion model takes the first weight + if ((*iterSP)->GetScoreProducerDescription() == "Distortion") { + Weights.resize(1); //take only the first element + } else { //lexicalized reordering model takes the other + Weights.erase(Weights.begin()); //remove the first element + } + // std::cerr << "this is the Distortion Score Producer -> " << (*iterSP)->GetScoreProducerDescription() << std::cerr; + // std::cerr << "this is the Distortion Score Producer; it has " << (*iterSP)->GetNumScoreComponents() << " weights"<< std::cerr; + // std::cerr << Weights << std::endl; + } else if (paramShortName == "tm") { + continue; + } + SetWeights(*iterSP, Weights); + } + + // std::cerr << "There are " << m_phraseDictionary.size() << " m_phraseDictionaryfeatures" << std::endl; + + const vector<float> WeightsTM = Scan<float>(m_parameter->GetParamShortName("tm")); + // std::cerr << "WeightsTM: " << WeightsTM << std::endl; + + const vector<float> WeightsLM = Scan<float>(m_parameter->GetParamShortName("lm")); + // std::cerr << "WeightsLM: " << WeightsLM << std::endl; + + size_t index_WeightTM = 0; + for(size_t i=0; i<transSystem.GetPhraseDictionaries().size(); ++i) { + PhraseDictionaryFeature &phraseDictionaryFeature = *m_phraseDictionary[i]; + + // std::cerr << "phraseDictionaryFeature.GetNumScoreComponents():" << phraseDictionaryFeature.GetNumScoreComponents() << std::endl; + // std::cerr << "phraseDictionaryFeature.GetNumInputScores():" << phraseDictionaryFeature.GetNumInputScores() << std::endl; + + vector<float> tmp_weights; + for(size_t j=0; j<phraseDictionaryFeature.GetNumScoreComponents(); ++j) + tmp_weights.push_back(WeightsTM[index_WeightTM++]); + + // std::cerr << tmp_weights << std::endl; + + SetWeights(&phraseDictionaryFeature, tmp_weights); + } + +} + +void StaticData::ReLoadBleuScoreFeatureParameter(float weight) +{ + //loop over ScoreProducers to update weights of BleuScoreFeature + const TranslationSystem &transSystem = GetTranslationSystem(TranslationSystem::DEFAULT); + + std::vector<const ScoreProducer*>::const_iterator iterSP; + for (iterSP = transSystem.GetFeatureFunctions().begin() ; iterSP != transSystem.GetFeatureFunctions().end() ; ++iterSP) { + std::string paramShortName = (*iterSP)->GetScoreProducerWeightShortName(); + if (paramShortName == "bl") { + SetWeight(*iterSP, weight); + break; + } + } +} + +// ScoreComponentCollection StaticData::GetAllWeightsScoreComponentCollection() const {} +// in ScoreComponentCollection.h + void StaticData::SetExecPath(const std::string &path) { /* diff --git a/moses/src/StaticData.h b/moses/src/StaticData.h index 15f4e0d59..4d69bf95d 100644 --- a/moses/src/StaticData.h +++ b/moses/src/StaticData.h @@ -37,7 +37,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #endif #include "TypeDef.h" -#include "ScoreIndexManager.h" #include "FactorCollection.h" #include "Parameter.h" #include "LM/Base.h" @@ -46,6 +45,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "DecodeGraph.h" #include "TranslationOptionList.h" #include "TranslationSystem.h" +#include "ScoreComponentCollection.h" namespace Moses { @@ -53,11 +53,23 @@ namespace Moses class InputType; class LexicalReordering; class GlobalLexicalModel; +class GlobalLexicalModelUnlimited; +class PhraseBoundaryFeature; class PhraseDictionaryFeature; +class SparsePhraseDictionaryFeature; +class PhrasePairFeature; +class BleuScoreFeature; +class PhraseLengthFeature; +class TargetWordInsertionFeature; +class SourceWordDeletionFeature; +class WordTranslationFeature; class GenerationDictionary; class DistortionScoreProducer; class DecodeStep; class UnknownWordPenaltyProducer; +class MetaScoreProducer; +class TargetBigramFeature; +class TargetNgramFeature; #ifdef HAVE_SYNLM class SyntacticLanguageModel; #endif @@ -78,22 +90,31 @@ protected: std::map<long,Phrase> m_constraints; std::vector<PhraseDictionaryFeature*> m_phraseDictionary; + std::vector<SparsePhraseDictionaryFeature*> m_sparsePhraseDictionary; std::vector<GenerationDictionary*> m_generationDictionary; Parameter *m_parameter; std::vector<FactorType> m_inputFactorOrder, m_outputFactorOrder; LMList m_languageModel; + ScoreComponentCollection m_allWeights; + std::vector<LexicalReordering*> m_reorderModels; + std::vector<GlobalLexicalModel*> m_globalLexicalModels; + std::vector<GlobalLexicalModelUnlimited*> m_globalLexicalModelsUnlimited; #ifdef HAVE_SYNLM SyntacticLanguageModel* m_syntacticLanguageModel; #endif - ScoreIndexManager m_scoreIndexManager; - std::vector<float> m_allWeights; - std::vector<LexicalReordering*> m_reorderModels; - std::vector<GlobalLexicalModel*> m_globalLexicalModels; std::vector<DecodeGraph*> m_decodeGraphs; std::vector<size_t> m_decodeGraphBackoff; // Initial = 0 = can be used when creating poss trans // Other = 1 = used to calculate LM score once all steps have been processed std::map<std::string, TranslationSystem> m_translationSystems; + TargetBigramFeature *m_targetBigramFeature; + std::vector<TargetNgramFeature*> m_targetNgramFeatures; + PhraseBoundaryFeature *m_phraseBoundaryFeature; + PhraseLengthFeature* m_phraseLengthFeature; + TargetWordInsertionFeature* m_targetWordInsertionFeature; + SourceWordDeletionFeature* m_sourceWordDeletionFeature; + std::vector<WordTranslationFeature*> m_wordTranslationFeatures; + std::vector<PhrasePairFeature*> m_phrasePairFeatures; float m_beamWidth, m_earlyDiscardingThreshold, @@ -143,6 +164,8 @@ protected: std::vector<WordPenaltyProducer*> m_wordPenaltyProducers; std::vector<DistortionScoreProducer *> m_distortionScoreProducers; UnknownWordPenaltyProducer *m_unknownWordPenaltyProducer; + MetaFeatureProducer *m_metaFeatureProducer; + BleuScoreFeature* m_bleuScoreFeature; bool m_reportSegmentation; bool m_reportAllFactors; bool m_reportAllFactorsNBest; @@ -163,6 +186,7 @@ protected: bool m_mbr; //! use MBR decoder bool m_useLatticeMBR; //! use MBR decoder + bool m_mira; // do mira training bool m_useConsensusDecoding; //! Use Consensus decoding (DeNero et al 2009) size_t m_mbrSize; //! number of translation candidates considered float m_mbrScale; //! scaling factor for computing marginal probability of candidate translation @@ -221,6 +245,8 @@ protected: StaticData(); + + void LoadPhraseBasedParameters(); void LoadChartDecodingParameters(); void LoadNonTerminals(); @@ -241,6 +267,17 @@ protected: bool LoadDecodeGraphs(); bool LoadLexicalReorderingModel(); bool LoadGlobalLexicalModel(); + bool LoadGlobalLexicalModelUnlimited(); + //References used for scoring feature (eg BleuScoreFeature) for online training + bool LoadReferences(); + bool LoadDiscrimLMFeature(); + bool LoadPhraseBoundaryFeature(); + bool LoadPhrasePairFeature(); + bool LoadPhraseLengthFeature(); + bool LoadTargetWordInsertionFeature(); + bool LoadSourceWordDeletionFeature(); + bool LoadWordTranslationFeature(); + void ReduceTransOptCache() const; bool m_continuePartialTranslation; @@ -253,16 +290,32 @@ public: } //! destructor ~StaticData(); + //! return static instance for use like global variable static const StaticData& Instance() { return s_instance; } + //! do NOT call unless you know what you're doing + static StaticData& InstanceNonConst() { + return s_instance; + } + + /** delete current static instance and replace with another. + * Used by gui front end + */ +#ifdef WIN32 + static void Reset() { + s_instance = StaticData(); + } +#endif + //! Load data into static instance. This function is required as LoadData() is not const static bool LoadDataStatic(Parameter *parameter, const std::string &execPath); //! Main function to load everything. Also initialize the Parameter object bool LoadData(Parameter *parameter); + void ClearData(); const PARAM_VEC &GetParam(const std::string ¶mName) const { return m_parameter->GetParam(paramName); @@ -275,7 +328,6 @@ public: return m_outputFactorOrder; } - inline bool GetSourceStartPosMattersForRecombination() const { return m_sourceStartPosMattersForRecombination; } @@ -306,6 +358,9 @@ public: bool IsWordDeletionEnabled() const { return m_wordDeletionEnabled; } + BleuScoreFeature* GetBleuScoreFeature() const { + return m_bleuScoreFeature; + } size_t GetMaxHypoStackSize() const { return m_maxHypoStackSize; } @@ -345,13 +400,6 @@ public: float GetTranslationOptionThreshold() const { return m_translationOptionThreshold; } - //! returns the total number of score components across all types, all factors - size_t GetTotalScoreComponents() const { - return m_scoreIndexManager.GetTotalNumberOfScores(); - } - const ScoreIndexManager& GetScoreIndexManager() const { - return m_scoreIndexManager; - } const TranslationSystem& GetTranslationSystem(std::string id) const { std::map<std::string, TranslationSystem>::const_iterator iter = @@ -419,7 +467,7 @@ public: return m_nBestFilePath; } bool IsNBestEnabled() const { - return (!m_nBestFilePath.empty()) || m_mbr || m_useLatticeMBR || m_outputSearchGraph || m_useConsensusDecoding || !m_latticeSamplesFilePath.empty() + return (!m_nBestFilePath.empty()) || m_mbr || m_useLatticeMBR || m_mira || m_outputSearchGraph || m_useConsensusDecoding || !m_latticeSamplesFilePath.empty() #ifdef HAVE_PROTOBUF || m_outputSearchGraphPB #endif @@ -441,7 +489,6 @@ public: } //! Sets the global score vector weights for a given ScoreProducer. - void SetWeightsForScoreProducer(const ScoreProducer* sp, const std::vector<float>& weights); InputTypeEnum GetInputType() const { return m_inputType; } @@ -454,14 +501,57 @@ public: LMList GetLMList() const { return m_languageModel; } + WordPenaltyProducer* GetFirstWordPenaltyProducer() const { + assert(m_wordPenaltyProducers.size() >= 1); + return m_wordPenaltyProducers[0]; + } + DistortionScoreProducer* GetDistortionScoreProducer() const { + assert(m_distortionScoreProducers.size() >= 1); + return m_distortionScoreProducers[0]; + } + MetaFeatureProducer* GetMetaFeatureProducer() const { + return m_metaFeatureProducer; + } + std::vector<LexicalReordering*> GetLexicalReorderModels() const { + return m_reorderModels; + } + std::vector<PhraseDictionaryFeature*> GetPhraseDictionaryModels() const { + return m_phraseDictionary; + } size_t GetNumInputScores() const { return m_numInputScores; } - const std::vector<float>& GetAllWeights() const { + const ScoreComponentCollection& GetAllWeights() const { return m_allWeights; } + void SetAllWeights(const ScoreComponentCollection& weights) { + m_allWeights = weights; + } + + //Weight for a single-valued feature + float GetWeight(const ScoreProducer* sp) const { + return m_allWeights.GetScoreForProducer(sp); + } + + //Weight for a single-valued feature + void SetWeight(const ScoreProducer* sp, float weight) ; + + + //Weights for feature with fixed number of values + std::vector<float> GetWeights(const ScoreProducer* sp) const { + return m_allWeights.GetScoresForProducer(sp); + } + + float GetSparseWeight(const FName& featureName) const { + return m_allWeights.GetSparseWeight(featureName); + } + + //Weights for feature with fixed number of values + void SetWeights(const ScoreProducer* sp, const std::vector<float>& weights); + + bool UseAlignmentInfo() const { return m_UseAlignmentInfo; } @@ -629,6 +719,13 @@ public: return m_continuePartialTranslation; } + void ReLoadParameter(); + void ReLoadBleuScoreFeatureParameter(float weight); + + Parameter* GetParameter() { + return m_parameter; + } + WordAlignmentSort GetWordAlignmentSort() const { return m_wordAlignmentSort; } diff --git a/moses/src/TargetBigramFeature.cpp b/moses/src/TargetBigramFeature.cpp new file mode 100644 index 000000000..a9ad2216b --- /dev/null +++ b/moses/src/TargetBigramFeature.cpp @@ -0,0 +1,99 @@ +#include "TargetBigramFeature.h" +#include "Phrase.h" +#include "TargetPhrase.h" +#include "Hypothesis.h" +#include "ScoreComponentCollection.h" + +namespace Moses { + +using namespace std; + +int TargetBigramState::Compare(const FFState& other) const { + const TargetBigramState& rhs = dynamic_cast<const TargetBigramState&>(other); + return Word::Compare(m_word,rhs.m_word); +} + +bool TargetBigramFeature::Load(const std::string &filePath) +{ + if (filePath == "*") return true; //allow all + ifstream inFile(filePath.c_str()); + if (!inFile) + { + return false; + } + + std::string line; + m_vocab.insert(BOS_); + m_vocab.insert(BOS_); + while (getline(inFile, line)) { + m_vocab.insert(line); + } + + inFile.close(); + return true; +} + +string TargetBigramFeature::GetScoreProducerWeightShortName(unsigned) const +{ + return "dlmb"; +} + +size_t TargetBigramFeature::GetNumInputScores() const +{ + return 0; +} + + +const FFState* TargetBigramFeature::EmptyHypothesisState(const InputType &/*input*/) const +{ + return new TargetBigramState(m_bos); +} + +FFState* TargetBigramFeature::Evaluate(const Hypothesis& cur_hypo, + const FFState* prev_state, + ScoreComponentCollection* accumulator) const +{ + const TargetBigramState* tbState = dynamic_cast<const TargetBigramState*>(prev_state); + CHECK(tbState); + + // current hypothesis target phrase + const Phrase& targetPhrase = cur_hypo.GetCurrTargetPhrase(); + if (targetPhrase.GetSize() == 0) { + return new TargetBigramState(*tbState); + } + + // extract all bigrams w1 w2 from current hypothesis + for (size_t i = 0; i < targetPhrase.GetSize(); ++i) { + const Factor* f1 = NULL; + if (i == 0) { + f1 = tbState->GetWord().GetFactor(m_factorType); + } else { + f1 = targetPhrase.GetWord(i-1).GetFactor(m_factorType); + } + const Factor* f2 = targetPhrase.GetWord(i).GetFactor(m_factorType); + const string& w1 = f1->GetString(); + const string& w2 = f2->GetString(); + + // skip bigrams if they don't belong to a given restricted vocabulary + if (m_vocab.size() && + (m_vocab.find(w1) == m_vocab.end() || m_vocab.find(w2) == m_vocab.end())) { + continue; + } + + string name(w1 +":"+w2); + accumulator->PlusEquals(this,name,1); + } + + if (cur_hypo.GetWordsBitmap().IsComplete()) { + const string& w1 = targetPhrase.GetWord(targetPhrase.GetSize()-1).GetFactor(m_factorType)->GetString(); + const string& w2 = EOS_; + if (m_vocab.empty() || (m_vocab.find(w1) != m_vocab.end())) { + string name(w1 +":"+w2); + accumulator->PlusEquals(this,name,1); + } + return NULL; + } + return new TargetBigramState(targetPhrase.GetWord(targetPhrase.GetSize()-1)); +} +} + diff --git a/moses/src/TargetBigramFeature.h b/moses/src/TargetBigramFeature.h new file mode 100644 index 000000000..76b4f6ef7 --- /dev/null +++ b/moses/src/TargetBigramFeature.h @@ -0,0 +1,64 @@ +#ifndef moses_TargetBigramFeature_h +#define moses_TargetBigramFeature_h + +#include <string> +#include <map> + +#include "FactorCollection.h" +#include "FeatureFunction.h" +#include "FFState.h" +#include "Word.h" + +namespace Moses +{ + +class TargetBigramState : public FFState { + public: + TargetBigramState(const Word& word): m_word(word) {} + const Word& GetWord() const {return m_word;} + virtual int Compare(const FFState& other) const; + + private: + Word m_word; +}; + +/** Sets the features of observed bigrams. + */ +class TargetBigramFeature : public StatefulFeatureFunction { +public: + TargetBigramFeature(FactorType factorType = 0): + StatefulFeatureFunction("dlmb", ScoreProducer::unlimited), + m_factorType(factorType) + { + FactorCollection& factorCollection = FactorCollection::Instance(); + const Factor* bosFactor = + factorCollection.AddFactor(Output,m_factorType,BOS_); + m_bos.SetFactor(m_factorType,bosFactor); + } + + + bool Load(const std::string &filePath); + + std::string GetScoreProducerWeightShortName(unsigned) const; + size_t GetNumInputScores() const; + + virtual const FFState* EmptyHypothesisState(const InputType &input) const; + + virtual FFState* Evaluate(const Hypothesis& cur_hypo, const FFState* prev_state, + ScoreComponentCollection* accumulator) const; + + virtual FFState* EvaluateChart( const ChartHypothesis& /* cur_hypo */, + int /* featureID */, + ScoreComponentCollection* ) const + { + abort(); + } +private: + FactorType m_factorType; + Word m_bos; + std::set<std::string> m_vocab; +}; + +} + +#endif // moses_TargetBigramFeature_h diff --git a/moses/src/TargetBigramFeatureTest.cpp b/moses/src/TargetBigramFeatureTest.cpp new file mode 100644 index 000000000..2ee7243bb --- /dev/null +++ b/moses/src/TargetBigramFeatureTest.cpp @@ -0,0 +1,189 @@ +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2010 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#include <cstdio> +#include <cstdlib> +#include <string> + +#include <boost/test/unit_test.hpp> + +#include "FactorCollection.h" +#include "Sentence.h" +#include "TargetBigramFeature.h" +#include "Word.h" + +#include "MockHypothesis.h" + +using namespace std; +using namespace Moses; + +namespace MosesTest +{ + +BOOST_AUTO_TEST_SUITE(target_bigram) + +static Word MakeWord(string text) { + FactorCollection &factorCollection = FactorCollection::Instance(); + const Factor* f = factorCollection.AddFactor(Input,0,text); + Word w; + w.SetFactor(0,f); + return w; +} + +class VocabFileFixture { + public: + template<class I> + VocabFileFixture(I begin, I end) + { + filename = string(tmpnam(NULL)); + ofstream out(filename.c_str()); + BOOST_CHECK(out); + for (I i = begin; i != end; ++i) + { + out << *i << endl; + } + out.close(); + } + + ~VocabFileFixture() + { + BOOST_CHECK(!remove(filename.c_str())); + } + + string filename; +}; + +/* +BOOST_AUTO_TEST_CASE(Test2) +{ + HypothesisFixture hypos; + cerr << hypos.empty() << ", " << *hypos.empty() << endl; + cerr << hypos.partial() << ", " << *hypos.partial() << endl; + cerr << hypos.full() << ", " << *hypos.full() << endl; + BOOST_CHECK(true); +} */ + +BOOST_AUTO_TEST_CASE(state_compare) +{ + Word w1 = MakeWord("w1"); + Word w2 = MakeWord("w2"); + TargetBigramState s1(w1); + TargetBigramState s2(w2); + BOOST_CHECK_EQUAL(s1.Compare(s1),0); + BOOST_CHECK_EQUAL(s2.Compare(s2),0); + BOOST_CHECK_NE(s1.Compare(s2),0); + BOOST_CHECK_NE(s2.Compare(s1),0); + +} + +BOOST_AUTO_TEST_CASE(load) +{ + TargetBigramFeature tbf; + string vocab[] = {"je", "ne", "pas"}; + VocabFileFixture vocabFixture(vocab,vocab+3); + BOOST_CHECK(tbf.Load(vocabFixture.filename)); + BOOST_CHECK(!tbf.Load("/gweugegyiegy")); +} + +BOOST_AUTO_TEST_CASE(score_components) +{ + TargetBigramFeature tbf; + BOOST_CHECK_EQUAL( + tbf.GetNumScoreComponents(), + ScoreProducer::unlimited); +} + +BOOST_AUTO_TEST_CASE(empty_hypo) +{ + Sentence s; + TargetBigramFeature tbf; + auto_ptr<const FFState> ffs(tbf.EmptyHypothesisState(s)); + BOOST_CHECK(ffs.get()); + TargetBigramState expected(MakeWord(BOS_)); + BOOST_CHECK_EQUAL(ffs->Compare(expected),0); +} + +//Test of evaluate() where a vocab is specified +BOOST_AUTO_TEST_CASE(evaluate_vocab) +{ + string vocab[] = {"i", "do"}; + VocabFileFixture vocabFile(vocab,vocab+2); + HypothesisFixture hypos; + TargetBigramFeature tbf; + BOOST_CHECK(tbf.Load(vocabFile.filename)); + TargetBigramState prevState(MakeWord("i")); + ScoreComponentCollection scc; + auto_ptr<FFState> currState( + tbf.Evaluate(*hypos.partial(), &prevState, &scc)); + + BOOST_CHECK_EQUAL(scc.GetScoreForProducer(&tbf, "i:do"),1); + BOOST_CHECK_EQUAL(scc.GetScoreForProducer(&tbf, "do:not"),0); + BOOST_CHECK(! currState->Compare(TargetBigramState(MakeWord("not")))); +} + +//Test of evaluate() where no vocab file is specified +BOOST_AUTO_TEST_CASE(evaluate_all) +{ + HypothesisFixture hypos; + TargetBigramFeature tbf; + BOOST_CHECK(tbf.Load("*")); + TargetBigramState prevState(MakeWord("i")); + ScoreComponentCollection scc; + auto_ptr<FFState> currState( + tbf.Evaluate(*hypos.partial(),&prevState,&scc)); + + BOOST_CHECK_EQUAL(scc.GetScoreForProducer(&tbf, "i:do"),1); + BOOST_CHECK_EQUAL(scc.GetScoreForProducer(&tbf, "do:not"),1); + BOOST_CHECK_EQUAL(scc.GetScoreForProducer(&tbf, "not:</s>"),0); + BOOST_CHECK(! currState->Compare(TargetBigramState(MakeWord("not")))); + +} + +BOOST_AUTO_TEST_CASE(evaluate_empty) +{ + HypothesisFixture hypos; + TargetBigramFeature tbf; + BOOST_CHECK(tbf.Load("*")); + auto_ptr<const FFState> prevState(tbf.EmptyHypothesisState(Sentence())); + ScoreComponentCollection scc; + auto_ptr<const FFState> currState( + tbf.Evaluate(*hypos.empty(),prevState.get(),&scc)); + BOOST_CHECK(! currState->Compare(*prevState)); +} + +BOOST_AUTO_TEST_CASE(evaluate_eos) +{ + HypothesisFixture hypos; + TargetBigramFeature tbf; + BOOST_CHECK(tbf.Load("*")); + TargetBigramState prevState(MakeWord("know")); + ScoreComponentCollection scc; + auto_ptr<const FFState> currState( + tbf.Evaluate(*hypos.full(),&prevState,&scc)); + + BOOST_CHECK_EQUAL(scc.GetScoreForProducer(&tbf, ".:</s>"),1); + BOOST_CHECK_EQUAL(scc.GetScoreForProducer(&tbf, "know:."),1); + + BOOST_CHECK(! currState.get()); +} + +BOOST_AUTO_TEST_SUITE_END() + +} + diff --git a/moses/src/TargetNgramFeature.cpp b/moses/src/TargetNgramFeature.cpp new file mode 100644 index 000000000..3fefdfba2 --- /dev/null +++ b/moses/src/TargetNgramFeature.cpp @@ -0,0 +1,436 @@ +#include "TargetNgramFeature.h" +#include "Phrase.h" +#include "TargetPhrase.h" +#include "Hypothesis.h" +#include "ScoreComponentCollection.h" +#include "ChartHypothesis.h" + +namespace Moses { + +using namespace std; + +int TargetNgramState::Compare(const FFState& other) const { + const TargetNgramState& rhs = dynamic_cast<const TargetNgramState&>(other); + int result; + if (m_words.size() == rhs.m_words.size()) { + for (size_t i = 0; i < m_words.size(); ++i) { + result = Word::Compare(m_words[i],rhs.m_words[i]); + if (result != 0) return result; + } + return 0; + } + else if (m_words.size() < rhs.m_words.size()) { + for (size_t i = 0; i < m_words.size(); ++i) { + result = Word::Compare(m_words[i],rhs.m_words[i]); + if (result != 0) return result; + } + return -1; + } + else { + for (size_t i = 0; i < rhs.m_words.size(); ++i) { + result = Word::Compare(m_words[i],rhs.m_words[i]); + if (result != 0) return result; + } + return 1; + } +} + +bool TargetNgramFeature::Load(const std::string &filePath) +{ + if (filePath == "*") return true; //allow all + ifstream inFile(filePath.c_str()); + if (!inFile) + { + return false; + } + + std::string line; + m_vocab.insert(BOS_); + m_vocab.insert(EOS_); + while (getline(inFile, line)) { + m_vocab.insert(line); + } + + inFile.close(); + return true; +} + +string TargetNgramFeature::GetScoreProducerWeightShortName(unsigned) const +{ + return "dlm"; +} + +size_t TargetNgramFeature::GetNumInputScores() const +{ + return 0; +} + +const FFState* TargetNgramFeature::EmptyHypothesisState(const InputType &/*input*/) const +{ + vector<Word> bos(1,m_bos); + return new TargetNgramState(bos); +} + +FFState* TargetNgramFeature::Evaluate(const Hypothesis& cur_hypo, + const FFState* prev_state, + ScoreComponentCollection* accumulator) const +{ + const TargetNgramState* tnState = static_cast<const TargetNgramState*>(prev_state); + assert(tnState); + + // current hypothesis target phrase + const Phrase& targetPhrase = cur_hypo.GetCurrTargetPhrase(); + if (targetPhrase.GetSize() == 0) return new TargetNgramState(*tnState); + + // extract all ngrams from current hypothesis + vector<Word> prev_words = tnState->GetWords(); + stringstream curr_ngram; + bool skip = false; + + // include lower order ngrams? + size_t smallest_n = m_n; + if (m_lower_ngrams) smallest_n = 1; + + for (size_t n = m_n; n >= smallest_n; --n) { // iterate over ngram size + for (size_t i = 0; i < targetPhrase.GetSize(); ++i) { +// const string& curr_w = targetPhrase.GetWord(i).GetFactor(m_factorType)->GetString(); + const string& curr_w = targetPhrase.GetWord(i).GetString(m_factorType); + + if (m_vocab.size() && (m_vocab.find(curr_w) == m_vocab.end())) continue; // skip ngrams + + if (n > 1) { + // can we build an ngram at this position? ("<s> this" --> cannot build 3gram at this position) + size_t pos_in_translation = cur_hypo.GetSize() - targetPhrase.GetSize() + i; + if (pos_in_translation < n - 2) continue; // need at least m_n - 1 words + + // how many words needed from previous state? + int from_prev_state = n - (i+1); + skip = false; + if (from_prev_state > 0) { + if (prev_words.size() < from_prev_state) { + // context is too short, make new state from previous state and target phrase + vector<Word> new_prev_words; + for (size_t i = 0; i < prev_words.size(); ++i) + new_prev_words.push_back(prev_words[i]); + for (size_t i = 0; i < targetPhrase.GetSize(); ++i) + new_prev_words.push_back(targetPhrase.GetWord(i)); + return new TargetNgramState(new_prev_words); + } + + // add words from previous state + for (size_t j = prev_words.size()-from_prev_state; j < prev_words.size() && !skip; ++j) + appendNgram(prev_words[j], skip, curr_ngram); + } + + // add words from current target phrase + int start = i - n + 1; // add m_n-1 previous words + if (start < 0) start = 0; // or less + for (size_t j = start; j < i && !skip; ++j) + appendNgram(targetPhrase.GetWord(j), skip, curr_ngram); + } + + if (!skip) { + curr_ngram << curr_w; + accumulator->PlusEquals(this,curr_ngram.str(),1); + } + curr_ngram.str(""); + } + } + + if (cur_hypo.GetWordsBitmap().IsComplete()) { + for (size_t n = m_n; n >= smallest_n; --n) { + stringstream last_ngram; + skip = false; + for (size_t i = cur_hypo.GetSize() - n + 1; i < cur_hypo.GetSize() && !skip; ++i) + appendNgram(cur_hypo.GetWord(i), skip, last_ngram); + + if (n > 1 && !skip) { + last_ngram << EOS_; + accumulator->PlusEquals(this, last_ngram.str(), 1); + } + } + return NULL; + } + + // prepare new state + vector<Word> new_prev_words; + if (targetPhrase.GetSize() >= m_n-1) { + // take subset of target words + for (size_t i = targetPhrase.GetSize() - m_n + 1; i < targetPhrase.GetSize(); ++i) + new_prev_words.push_back(targetPhrase.GetWord(i)); + } + else { + // take words from previous state and from target phrase + int from_prev_state = m_n - 1 - targetPhrase.GetSize(); + for (size_t i = prev_words.size()-from_prev_state; i < prev_words.size(); ++i) + new_prev_words.push_back(prev_words[i]); + for (size_t i = 0; i < targetPhrase.GetSize(); ++i) + new_prev_words.push_back(targetPhrase.GetWord(i)); + } + return new TargetNgramState(new_prev_words); +} + +void TargetNgramFeature::appendNgram(const Word& word, bool& skip, stringstream &ngram) const { +// const string& w = word.GetFactor(m_factorType)->GetString(); + const string& w = word.GetString(m_factorType); + if (m_vocab.size() && (m_vocab.find(w) == m_vocab.end())) skip = true; + else { + ngram << w; + ngram << ":"; + } +} + +FFState* TargetNgramFeature::EvaluateChart(const ChartHypothesis& cur_hypo, int featureId, ScoreComponentCollection* accumulator) const +{ + vector<const Word*> contextFactor; + contextFactor.reserve(m_n); + + // get index map for underlying hypotheses + const AlignmentInfo::NonTermIndexMap &nonTermIndexMap = + cur_hypo.GetCurrTargetPhrase().GetAlignmentInfo().GetNonTermIndexMap(); + + // loop over rule + bool makePrefix = false; + bool makeSuffix = false; + bool collectForPrefix = true; + size_t prefixTerminals = 0; + size_t suffixTerminals = 0; + bool onlyTerminals = true; + bool prev_is_NT = false; + size_t prev_subPhraseLength = 0; + for (size_t phrasePos = 0; phrasePos < cur_hypo.GetCurrTargetPhrase().GetSize(); phrasePos++) + { + // consult rule for either word or non-terminal + const Word &word = cur_hypo.GetCurrTargetPhrase().GetWord(phrasePos); +// cerr << "word: " << word << endl; + + // regular word + if (!word.IsNonTerminal()) { + contextFactor.push_back(&word); + prev_is_NT = false; + + if (phrasePos==0) + makePrefix = true; + if (phrasePos==cur_hypo.GetCurrTargetPhrase().GetSize()-1 || prev_is_NT) + makeSuffix = true; + + // beginning/end of sentence symbol <s>,</s>? + string factorZero = word.GetString(0); + if (factorZero.compare("<s>") == 0) + prefixTerminals++; + // end of sentence symbol </s>? + else if (factorZero.compare("</s>") == 0) + suffixTerminals++; + // everything else + else { + stringstream ngram; + ngram << m_baseName; + if (m_factorType == 0) + ngram << factorZero; + else + ngram << word.GetString(m_factorType); + accumulator->SparsePlusEquals(ngram.str(), 1); + + if (collectForPrefix) + prefixTerminals++; + else + suffixTerminals++; + } + } + + // non-terminal, add phrase from underlying hypothesis + else if (m_n > 1) + { + // look up underlying hypothesis + size_t nonTermIndex = nonTermIndexMap[phrasePos]; + const ChartHypothesis *prevHypo = cur_hypo.GetPrevHypo(nonTermIndex); + + const TargetNgramChartState* prevState = + static_cast<const TargetNgramChartState*>(prevHypo->GetFFState(featureId)); + size_t subPhraseLength = prevState->GetNumTargetTerminals(); + + // special case: rule starts with non-terminal + if (phrasePos == 0) { + if (subPhraseLength == 1) { + makePrefix = true; + ++prefixTerminals; + + const Word &word = prevState->GetSuffix().GetWord(0); +// cerr << "NT0 --> : " << word << endl; + contextFactor.push_back(&word); + } + else { + onlyTerminals = false; + collectForPrefix = false; + int suffixPos = prevState->GetSuffix().GetSize() - (m_n-1); + if (suffixPos < 0) suffixPos = 0; // push all words if less than order + for(;(size_t)suffixPos < prevState->GetSuffix().GetSize(); suffixPos++) + { + const Word &word = prevState->GetSuffix().GetWord(suffixPos); +// cerr << "NT0 --> : " << word << endl; + contextFactor.push_back(&word); + } + } + } + + // internal non-terminal + else + { + // push its prefix + for(size_t prefixPos = 0; prefixPos < m_n-1 + && prefixPos < subPhraseLength; prefixPos++) + { + const Word &word = prevState->GetPrefix().GetWord(prefixPos); +// cerr << "NT --> " << word << endl; + contextFactor.push_back(&word); + } + + if (subPhraseLength==1) { + if (collectForPrefix) + ++prefixTerminals; + else + ++suffixTerminals; + + if (phrasePos == cur_hypo.GetCurrTargetPhrase().GetSize()-1) + makeSuffix = true; + } + else { + onlyTerminals = false; + collectForPrefix = true; + + // check if something follows this NT + bool wordFollowing = (phrasePos < cur_hypo.GetCurrTargetPhrase().GetSize() - 1)? true : false; + + // check if we are dealing with a large sub-phrase + if (wordFollowing && subPhraseLength > m_n - 1) + { + // clear up pending ngrams + MakePrefixNgrams(contextFactor, accumulator, prefixTerminals); + contextFactor.clear(); + makePrefix = false; + makeSuffix = true; + collectForPrefix = false; + prefixTerminals = 0; + suffixTerminals = 0; + + // push its suffix + size_t remainingWords = (remainingWords > m_n-1) ? m_n-1 : subPhraseLength - (m_n-1); + for(size_t suffixPos = 0; suffixPos < prevState->GetSuffix().GetSize(); suffixPos++) { + const Word &word = prevState->GetSuffix().GetWord(suffixPos); +// cerr << "NT --> : " << word << endl; + contextFactor.push_back(&word); + } + } + // subphrase can be used as suffix and as prefix for the next part + else if (wordFollowing && subPhraseLength == m_n - 1) + { + // clear up pending ngrams + MakePrefixNgrams(contextFactor, accumulator, prefixTerminals); + makePrefix = false; + makeSuffix = true; + collectForPrefix = false; + prefixTerminals = 0; + suffixTerminals = 0; + } + else if (prev_is_NT && prev_subPhraseLength > 1 && subPhraseLength > 1) { + // two NTs in a row: make transition + MakePrefixNgrams(contextFactor, accumulator, 1, m_n-2); + MakeSuffixNgrams(contextFactor, accumulator, 1, m_n-2); + makePrefix = false; + makeSuffix = false; + collectForPrefix = false; + prefixTerminals = 0; + suffixTerminals = 0; + + // remove duplicates + stringstream curr_ngram; + curr_ngram << m_baseName; + curr_ngram << (*contextFactor[m_n-2]).GetString(m_factorType); + curr_ngram << ":"; + curr_ngram << (*contextFactor[m_n-1]).GetString(m_factorType); + accumulator->SparseMinusEquals(curr_ngram.str(),1); + } + } + } + prev_is_NT = true; + prev_subPhraseLength = subPhraseLength; + } + } + + if (m_n > 1) { + if (onlyTerminals) { + MakePrefixNgrams(contextFactor, accumulator, prefixTerminals-1); + } + else { + if (makePrefix) + MakePrefixNgrams(contextFactor, accumulator, prefixTerminals); + if (makeSuffix) + MakeSuffixNgrams(contextFactor, accumulator, suffixTerminals); + + // remove duplicates + size_t size = contextFactor.size(); + if (makePrefix && makeSuffix && (size <= m_n)) { + stringstream curr_ngram; + curr_ngram << m_baseName; + for (size_t i = 0; i < size; ++i) { + curr_ngram << (*contextFactor[i]).GetString(m_factorType); + if (i < size-1) + curr_ngram << ":"; + } + accumulator->SparseMinusEquals(curr_ngram.str(), 1); + } + } + } + +// cerr << endl; + return new TargetNgramChartState(cur_hypo, featureId, m_n); +} + +void TargetNgramFeature::MakePrefixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, size_t numberOfStartPos, size_t offset) const { + stringstream ngram; + size_t size = contextFactor.size(); + for (size_t k = 0; k < numberOfStartPos; ++k) { + size_t max_end = (size < m_n+k+offset)? size: m_n+k+offset; + for (size_t end_pos = 1+k+offset; end_pos < max_end; ++end_pos) { + ngram << m_baseName; + for (size_t i=k+offset; i <= end_pos; ++i) { + if (i > k+offset) + ngram << ":"; + string factorZero = (*contextFactor[i]).GetString(0); + if (m_factorType == 0 || factorZero.compare("<s>") == 0 || factorZero.compare("</s>") == 0) + ngram << factorZero; + else + ngram << (*contextFactor[i]).GetString(m_factorType); + const Word w = *contextFactor[i]; + } +// cerr << "p-ngram: " << ngram.str() << endl; + accumulator->SparsePlusEquals(ngram.str(), 1); + ngram.str(""); + } + } +} + +void TargetNgramFeature::MakeSuffixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, size_t numberOfEndPos, size_t offset) const { + stringstream ngram; + for (size_t k = 0; k < numberOfEndPos; ++k) { + size_t end_pos = contextFactor.size()-1-k-offset; + for (int start_pos=end_pos-1; (start_pos >= 0) && (end_pos-start_pos < m_n); --start_pos) { + ngram << m_baseName; + for (size_t j=start_pos; j <= end_pos; ++j){ + string factorZero = (*contextFactor[j]).GetString(0); + if (m_factorType == 0 || factorZero.compare("<s>") == 0 || factorZero.compare("</s>") == 0) + ngram << factorZero; + else + ngram << (*contextFactor[j]).GetString(m_factorType); + if (j < end_pos) + ngram << ":"; + } +// cerr << "s-ngram: " << ngram.str() << endl; + accumulator->SparsePlusEquals(ngram.str(), 1); + ngram.str(""); + } + } +} + +} + diff --git a/moses/src/TargetNgramFeature.h b/moses/src/TargetNgramFeature.h new file mode 100644 index 000000000..681e7d6aa --- /dev/null +++ b/moses/src/TargetNgramFeature.h @@ -0,0 +1,234 @@ +#ifndef moses_TargetNgramFeature_h +#define moses_TargetNgramFeature_h + +#include <string> +#include <map> + +#include "FactorCollection.h" +#include "FeatureFunction.h" +#include "FFState.h" +#include "Word.h" + +#include "LM/SingleFactor.h" +#include "ChartHypothesis.h" +#include "ChartManager.h" + +namespace Moses +{ + +class TargetNgramState : public FFState { + public: + TargetNgramState(std::vector<Word> &words): m_words(words) {} + const std::vector<Word> GetWords() const {return m_words;} + virtual int Compare(const FFState& other) const; + + private: + std::vector<Word> m_words; +}; + +class TargetNgramChartState : public FFState +{ +private: + Phrase m_contextPrefix, m_contextSuffix; + + size_t m_numTargetTerminals; // This isn't really correct except for the surviving hypothesis + + size_t m_startPos, m_endPos, m_inputSize; + + /** Construct the prefix string of up to specified size + * \param ret prefix string + * \param size maximum size (typically max lm context window) + */ + size_t CalcPrefix(const ChartHypothesis &hypo, const int featureId, Phrase &ret, size_t size) const + { + const TargetPhrase &target = hypo.GetCurrTargetPhrase(); + const AlignmentInfo::NonTermIndexMap &nonTermIndexMap = + target.GetAlignmentInfo().GetNonTermIndexMap(); + + // loop over the rule that is being applied + for (size_t pos = 0; pos < target.GetSize(); ++pos) { + const Word &word = target.GetWord(pos); + + // for non-terminals, retrieve it from underlying hypothesis + if (word.IsNonTerminal()) { + size_t nonTermInd = nonTermIndexMap[pos]; + const ChartHypothesis *prevHypo = hypo.GetPrevHypo(nonTermInd); + size = static_cast<const TargetNgramChartState*>(prevHypo->GetFFState(featureId))->CalcPrefix(*prevHypo, featureId, ret, size); +// Phrase phrase = static_cast<const TargetNgramChartState*>(prevHypo->GetFFState(featureId))->GetPrefix(); +// size = phrase.GetSize(); + } + // for words, add word + else { + ret.AddWord(word); + size--; + } + + // finish when maximum length reached + if (size==0) + break; + } + + return size; + } + + /** Construct the suffix phrase of up to specified size + * will always be called after the construction of prefix phrase + * \param ret suffix phrase + * \param size maximum size of suffix + */ + size_t CalcSuffix(const ChartHypothesis &hypo, int featureId, Phrase &ret, size_t size) const + { + size_t prefixSize = m_contextPrefix.GetSize(); + assert(prefixSize <= m_numTargetTerminals); + + // special handling for small hypotheses + // does the prefix match the entire hypothesis string? -> just copy prefix + if (prefixSize == m_numTargetTerminals) { + size_t maxCount = std::min(prefixSize, size); + size_t pos= prefixSize - 1; + + for (size_t ind = 0; ind < maxCount; ++ind) { + const Word &word = m_contextPrefix.GetWord(pos); + ret.PrependWord(word); + --pos; + } + + size -= maxCount; + return size; + } + // construct suffix analogous to prefix + else { + const TargetPhrase targetPhrase = hypo.GetCurrTargetPhrase(); + const AlignmentInfo::NonTermIndexMap &nonTermIndexMap = + targetPhrase.GetAlignmentInfo().GetNonTermIndexMap(); + for (int pos = (int) targetPhrase.GetSize() - 1; pos >= 0 ; --pos) { + const Word &word = targetPhrase.GetWord(pos); + + if (word.IsNonTerminal()) { + size_t nonTermInd = nonTermIndexMap[pos]; + const ChartHypothesis *prevHypo = hypo.GetPrevHypo(nonTermInd); + size = static_cast<const TargetNgramChartState*>(prevHypo->GetFFState(featureId))->CalcSuffix(*prevHypo, featureId, ret, size); + } + else { + ret.PrependWord(word); + size--; + } + + if (size==0) + break; + } + + return size; + } + } + +public: + TargetNgramChartState(const ChartHypothesis &hypo, int featureId, size_t order) + :m_contextPrefix(order - 1), + m_contextSuffix(order - 1) + { + m_numTargetTerminals = hypo.GetCurrTargetPhrase().GetNumTerminals(); + const WordsRange range = hypo.GetCurrSourceRange(); + m_startPos = range.GetStartPos(); + m_endPos = range.GetEndPos(); + m_inputSize = hypo.GetManager().GetSource().GetSize(); + + const std::vector<const ChartHypothesis*> prevHypos = hypo.GetPrevHypos(); + for (std::vector<const ChartHypothesis*>::const_iterator i = prevHypos.begin(); i != prevHypos.end(); ++i) { + // keep count of words (= length of generated string) + m_numTargetTerminals += static_cast<const TargetNgramChartState*>((*i)->GetFFState(featureId))->GetNumTargetTerminals(); + } + + CalcPrefix(hypo, featureId, m_contextPrefix, order - 1); + CalcSuffix(hypo, featureId, m_contextSuffix, order - 1); + } + + size_t GetNumTargetTerminals() const { + return m_numTargetTerminals; + } + + const Phrase &GetPrefix() const { + return m_contextPrefix; + } + const Phrase &GetSuffix() const { + return m_contextSuffix; + } + + int Compare(const FFState& o) const { + const TargetNgramChartState &other = + static_cast<const TargetNgramChartState &>( o ); + + // prefix + if (m_startPos > 0) // not for "<s> ..." + { + int ret = GetPrefix().Compare(other.GetPrefix()); + if (ret != 0) + return ret; + } + + if (m_endPos < m_inputSize - 1)// not for "... </s>" + { + int ret = GetSuffix().Compare(other.GetSuffix()); + if (ret != 0) + return ret; + } + return 0; + } +}; + +/** Sets the features of observed ngrams. + */ +class TargetNgramFeature : public StatefulFeatureFunction { +public: + TargetNgramFeature(FactorType factorType = 0, size_t n = 3, bool lower_ngrams = true): + StatefulFeatureFunction("dlm", ScoreProducer::unlimited), + m_factorType(factorType), + m_n(n), + m_lower_ngrams(lower_ngrams), + m_sparseProducerWeight(1) + { + FactorCollection& factorCollection = FactorCollection::Instance(); + const Factor* bosFactor = factorCollection.AddFactor(Output,m_factorType,BOS_); + m_bos.SetFactor(m_factorType,bosFactor); + m_baseName = GetScoreProducerDescription(); + m_baseName.append("_"); + } + + bool Load(const std::string &filePath); + + std::string GetScoreProducerWeightShortName(unsigned) const; + size_t GetNumInputScores() const; + + void SetSparseProducerWeight(float weight) { m_sparseProducerWeight = weight; } + float GetSparseProducerWeight() const { return m_sparseProducerWeight; } + + virtual const FFState* EmptyHypothesisState(const InputType &input) const; + + virtual FFState* Evaluate(const Hypothesis& cur_hypo, const FFState* prev_state, + ScoreComponentCollection* accumulator) const; + + virtual FFState* EvaluateChart(const ChartHypothesis& cur_hypo, int featureId, + ScoreComponentCollection* accumulator) const; + +private: + FactorType m_factorType; + Word m_bos; + std::set<std::string> m_vocab; + size_t m_n; + bool m_lower_ngrams; + + // additional weight that all sparse weights are scaled with + float m_sparseProducerWeight; + + std::string m_baseName; + + void appendNgram(const Word& word, bool& skip, std::stringstream& ngram) const; + void MakePrefixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, + size_t numberOfStartPos = 1, size_t offset = 0) const; + void MakeSuffixNgrams(std::vector<const Word*> &contextFactor, ScoreComponentCollection* accumulator, + size_t numberOfEndPos = 1, size_t offset = 0) const; +}; + +} + +#endif // moses_TargetNgramFeature_h diff --git a/moses/src/TargetPhrase.cpp b/moses/src/TargetPhrase.cpp index e5eaac809..20cb26224 100644 --- a/moses/src/TargetPhrase.cpp +++ b/moses/src/TargetPhrase.cpp @@ -29,19 +29,20 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "GenerationDictionary.h" #include "LM/Base.h" #include "StaticData.h" -#include "ScoreIndexManager.h" #include "LMList.h" #include "ScoreComponentCollection.h" #include "Util.h" #include "DummyScoreProducers.h" #include "AlignmentInfoCollection.h" +#include <boost/algorithm/string.hpp> + using namespace std; namespace Moses { -TargetPhrase::TargetPhrase( std::string out_string) - :Phrase(0),m_transScore(0.0), m_fullScore(0.0), m_sourcePhrase(0) +TargetPhrase::TargetPhrase(std::string out_string) + :Phrase(0),m_transScore(0.0), m_fullScore(0.0), m_sourcePhrase(0) , m_alignmentInfo(&AlignmentInfoCollection::Instance().GetEmptyAlignmentInfo()) { @@ -55,7 +56,7 @@ TargetPhrase::TargetPhrase() :Phrase(ARRAY_SIZE_INCR) , m_transScore(0.0) , m_fullScore(0.0) - , m_sourcePhrase(0) + ,m_sourcePhrase(0) , m_alignmentInfo(&AlignmentInfoCollection::Instance().GetEmptyAlignmentInfo()) { } @@ -69,10 +70,6 @@ TargetPhrase::TargetPhrase(const Phrase &phrase) { } -TargetPhrase::~TargetPhrase() -{ -} - void TargetPhrase::SetScore(const TranslationSystem* system) { // used when creating translations of unknown words: @@ -93,32 +90,23 @@ void TargetPhrase::WriteToRulePB(hgmert::Rule* pb) const void TargetPhrase::SetScore(float score) { - //we use an existing score producer to figure out information for score setting (number of scores and weights) - //TODO: is this a good idea? - // Assume the default system. - const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT); - const ScoreProducer* prod = system.GetPhraseDictionaries()[0]; - - //get the weight list - unsigned int id = prod->GetScoreBookkeepingID(); - - const vector<float> &allWeights = StaticData::Instance().GetAllWeights(); - - size_t beginIndex = StaticData::Instance().GetScoreIndexManager().GetBeginIndex(id); - size_t endIndex = StaticData::Instance().GetScoreIndexManager().GetEndIndex(id); - - vector<float> weights; - - std::copy(allWeights.begin() +beginIndex, allWeights.begin() + endIndex,std::back_inserter(weights)); - - //find out how many items are in the score vector for this producer - size_t numScores = prod->GetNumScoreComponents(); - - //divide up the score among all of the score vectors - vector <float> scoreVector(numScores,score/numScores); - - //Now we have what we need to call the full SetScore method - SetScore(prod,scoreVector,weights,system.GetWeightWordPenalty(),system.GetLanguageModels()); + //we use an existing score producer to figure out information for score setting (number of scores and weights) + //TODO: is this a good idea? + // Assume the default system. + const TranslationSystem& system = StaticData::Instance().GetTranslationSystem(TranslationSystem::DEFAULT); + const ScoreProducer* prod = system.GetPhraseDictionaries()[0]; + + vector<float> weights = StaticData::Instance().GetWeights(prod); + + + //find out how many items are in the score vector for this producer + size_t numScores = prod->GetNumScoreComponents(); + + //divide up the score among all of the score vectors + vector <float> scoreVector(numScores,score/numScores); + + //Now we have what we need to call the full SetScore method + SetScore(prod, scoreVector, ScoreComponentCollection(), weights, system.GetWeightWordPenalty(), system.GetLanguageModels()); } /** @@ -127,28 +115,23 @@ void TargetPhrase::SetScore(float score) */ void TargetPhrase::SetScore(const TranslationSystem* system, const Scores &scoreVector) { - //we use an existing score producer to figure out information for score setting (number of scores and weights) - - const ScoreProducer* prod = system->GetPhraseDictionaries()[0]; + //we use an existing score producer to figure out information for score setting (number of scores and weights) - //get the weight list - unsigned int id = prod->GetScoreBookkeepingID(); - const vector<float> &allWeights = StaticData::Instance().GetAllWeights(); - size_t beginIndex = StaticData::Instance().GetScoreIndexManager().GetBeginIndex(id); - size_t endIndex = StaticData::Instance().GetScoreIndexManager().GetEndIndex(id); - vector<float> weights; - std::copy(allWeights.begin() +beginIndex, allWeights.begin() + endIndex,std::back_inserter(weights)); + const ScoreProducer* prod = system->GetPhraseDictionaries()[0]; - //expand the input weight vector - CHECK(scoreVector.size() <= prod->GetNumScoreComponents()); - Scores sizedScoreVector = scoreVector; - sizedScoreVector.resize(prod->GetNumScoreComponents(),0.0f); + vector<float> weights = StaticData::Instance().GetWeights(prod); + + //expand the input weight vector + CHECK(scoreVector.size() <= prod->GetNumScoreComponents()); + Scores sizedScoreVector = scoreVector; + sizedScoreVector.resize(prod->GetNumScoreComponents(),0.0f); - SetScore(prod,sizedScoreVector,weights,system->GetWeightWordPenalty(),system->GetLanguageModels()); + SetScore(prod,sizedScoreVector, ScoreComponentCollection(),weights,system->GetWeightWordPenalty(),system->GetLanguageModels()); } void TargetPhrase::SetScore(const ScoreProducer* translationScoreProducer, const Scores &scoreVector, + const ScoreComponentCollection &sparseScoreVector, const vector<float> &weightT, float weightWP, const LMList &languageModels) { @@ -157,6 +140,7 @@ void TargetPhrase::SetScore(const ScoreProducer* translationScoreProducer, m_transScore = std::inner_product(scoreVector.begin(), scoreVector.end(), weightT.begin(), 0.0f); m_scoreBreakdown.PlusEquals(translationScoreProducer, scoreVector); + m_scoreBreakdown.PlusEquals(sparseScoreVector); // Replicated from TranslationOptions.cpp float totalNgramScore = 0; @@ -195,7 +179,7 @@ void TargetPhrase::SetScore(const ScoreProducer* translationScoreProducer, } m_fullScore = m_transScore + totalFullScore + totalOOVScore - - (this->GetSize() * weightWP); // word penalty + - (this->GetSize() * weightWP); // word penalty } void TargetPhrase::SetScoreChart(const ScoreProducer* translationScoreProducer, @@ -204,9 +188,8 @@ void TargetPhrase::SetScoreChart(const ScoreProducer* translationScoreProducer, ,const LMList &languageModels ,const WordPenaltyProducer* wpProducer) { - CHECK(weightT.size() == scoreVector.size()); - + // calc average score if non-best m_transScore = std::inner_product(scoreVector.begin(), scoreVector.end(), weightT.begin(), 0.0f); m_scoreBreakdown.PlusEquals(translationScoreProducer, scoreVector); @@ -272,7 +255,6 @@ void TargetPhrase::SetWeights(const ScoreProducer* translationScoreProducer, con addition to the usual phrase translation scaling factors) the input weight factor as last element */ - m_transScore = m_scoreBreakdown.PartialInnerProduct(translationScoreProducer, weightT); } @@ -330,12 +312,42 @@ void TargetPhrase::SetAlignmentInfo(const StringPiece &alignString) SetAlignmentInfo(alignmentInfo); } + + +void TargetPhrase::SetAlignmentInfo(const StringPiece &alignString, Phrase &sourcePhrase) +{ + std::vector<std::string> alignPoints; + boost::split(alignPoints, alignString, boost::is_any_of("\t ")); + int indicator[alignPoints.size()]; + int index = 0; + + set<pair<size_t,size_t> > alignmentInfo; + for (util::TokenIter<util::AnyCharacter, true> token(alignString, util::AnyCharacter(" \t")); token; ++token) { + util::TokenIter<util::AnyCharacter, false> dash(*token, util::AnyCharacter("-")); + MosesShouldUseExceptions(dash); + size_t sourcePos = boost::lexical_cast<size_t>(*dash++); + MosesShouldUseExceptions(dash); + size_t targetPos = boost::lexical_cast<size_t>(*dash++); + MosesShouldUseExceptions(!dash); + + alignmentInfo.insert(pair<size_t,size_t>(sourcePos, targetPos)); + indicator[index++] = sourcePhrase.GetWord(sourcePos).IsNonTerminal() ? 1: 0; + } + + SetAlignmentInfo(alignmentInfo, indicator); +} + void TargetPhrase::SetAlignmentInfo(const std::set<std::pair<size_t,size_t> > &alignmentInfo) { - m_alignmentInfo = AlignmentInfoCollection::Instance().Add(alignmentInfo); + m_alignmentInfo = AlignmentInfoCollection::Instance().Add(alignmentInfo); } +void TargetPhrase::SetAlignmentInfo(const std::set<std::pair<size_t,size_t> > &alignmentInfo, int* indicator) +{ + m_alignmentInfo = AlignmentInfoCollection::Instance().Add(alignmentInfo, indicator); +} + TO_STRING_BODY(TargetPhrase); std::ostream& operator<<(std::ostream& os, const TargetPhrase& tp) @@ -346,5 +358,35 @@ std::ostream& operator<<(std::ostream& os, const TargetPhrase& tp) return os; } +void TargetPhrase::SetRuleCount(const StringPiece &ruleCountString, std::vector<float> &scoreVector) { + std::vector<std::string> tokens; + boost::split(tokens, ruleCountString, boost::is_any_of("\t ")); + + if (tokens.size() == 2) { + // TODO: if no third column is provided, do we have to take smoothing into account (consolidate.cpp)? + // infer rule counts from target counts + float targetCount = 0, sourceCount = 0; + float p_f_given_e = 0, p_e_given_f = 0; + p_f_given_e = scoreVector[0]; + if (scoreVector.size() >= 5) { + p_f_given_e = scoreVector[0]; + p_e_given_f = scoreVector[2]; + } + else { + if (scoreVector.size() >= 1 ) p_f_given_e = scoreVector[0]; +// std::cerr << "Warning: possibly wrong format of phrase translation scores, number of scores: " << scoreVector.size() << endl; + } + + targetCount = Scan<float>(tokens[0]); + sourceCount = Scan<float>(tokens[1]); + float ruleCount = p_f_given_e * targetCount; + //float ruleCount2 = p_e_given_f * sourceCount; // could use this to double-check the counts + m_ruleCount = floor(ruleCount + 0.5); + } + else if (tokens.size() == 3) { + m_ruleCount = Scan<float>(tokens[2]); + } +} + } diff --git a/moses/src/TargetPhrase.h b/moses/src/TargetPhrase.h index 68c3bb903..24c69c529 100644 --- a/moses/src/TargetPhrase.h +++ b/moses/src/TargetPhrase.h @@ -48,21 +48,20 @@ class TargetPhrase: public Phrase { friend std::ostream& operator<<(std::ostream&, const TargetPhrase&); protected: - float m_transScore; - float m_fullScore; - ScoreComponentCollection m_scoreBreakdown; + float m_transScore; + float m_fullScore; + ScoreComponentCollection m_scoreBreakdown; - // in case of confusion net, ptr to source phrase - Phrase const* m_sourcePhrase; - - const AlignmentInfo *m_alignmentInfo; - Word m_lhsTarget; + // in case of confusion net, ptr to source phrase + Phrase m_sourcePhrase; + const AlignmentInfo* m_alignmentInfo; + Word m_lhsTarget; + size_t m_ruleCount; public: TargetPhrase(); TargetPhrase(std::string out_string); - TargetPhrase(const Phrase &); - ~TargetPhrase(); + TargetPhrase(const Phrase &targetPhrase); //! used by the unknown word handler- these targets //! don't have a translation score, so wp is the only thing used @@ -88,6 +87,7 @@ public: */ void SetScore(const ScoreProducer* translationScoreProducer, const Scores &scoreVector, + const ScoreComponentCollection &sparseScoreVector, const std::vector<float> &weightT, float weightWP, const LMList &languageModels); @@ -129,43 +129,75 @@ public: inline void SetFutureScore(float fullScore) { m_fullScore = fullScore; } - inline const ScoreComponentCollection &GetScoreBreakdown() const { - return m_scoreBreakdown; - } - - //! TODO - why is this needed and is it set correctly by every phrase dictionary class ? should be set in constructor - void SetSourcePhrase(Phrase const* p) { - m_sourcePhrase=p; - } - Phrase const* GetSourcePhrase() const { - return m_sourcePhrase; - } - - void SetTargetLHS(const Word &lhs) { - m_lhsTarget = lhs; - } + inline const ScoreComponentCollection &GetScoreBreakdown() const + { + return m_scoreBreakdown; + } + + //TODO: Probably shouldn't copy this, but otherwise ownership is unclear + void SetSourcePhrase(const Phrase& p) + { + m_sourcePhrase=p; + } + const Phrase& GetSourcePhrase() const + { + return m_sourcePhrase; + } + + void SetTargetLHS(const Word &lhs) + { m_lhsTarget = lhs; } + const Word &GetTargetLHS() const + { return m_lhsTarget; } + Word &MutableTargetLHS() { return m_lhsTarget; } - const Word &GetTargetLHS() const { - return m_lhsTarget; - } void SetAlignmentInfo(const StringPiece &alignString); + void SetAlignmentInfo(const StringPiece &alignString, Phrase &sourcePhrase); void SetAlignmentInfo(const std::set<std::pair<size_t,size_t> > &alignmentInfo); + void SetAlignmentInfo(const std::set<std::pair<size_t,size_t> > &alignmentInfo, int* indicator); void SetAlignmentInfo(const AlignmentInfo *alignmentInfo) { m_alignmentInfo = alignmentInfo; } - - const AlignmentInfo &GetAlignmentInfo() const { - return *m_alignmentInfo; - } + + const AlignmentInfo &GetAlignmentInfo() const + { return *m_alignmentInfo; } + + void SetRuleCount(const StringPiece &ruleCountString, std::vector<float> &scoreVector); + size_t GetRuleCount() const { return m_ruleCount; } TO_STRING(); + + }; std::ostream& operator<<(std::ostream&, const TargetPhrase&); +/** + * Hasher that looks at source and target phrase. + **/ +struct RuleHash +{ + inline size_t operator()(const TargetPhrase& targetPhrase) const + { + size_t seed = 0; + boost::hash_combine(seed, targetPhrase); + boost::hash_combine(seed, targetPhrase.GetSourcePhrase()); + return seed; + } +}; + +struct RuleComparator +{ + inline bool operator()(const TargetPhrase& lhs, const TargetPhrase& rhs) const + { + return lhs.Compare(rhs) == 0 && + lhs.GetSourcePhrase().Compare(rhs.GetSourcePhrase()) == 0; + } + +}; + } #endif diff --git a/moses/src/TargetWordInsertionFeature.cpp b/moses/src/TargetWordInsertionFeature.cpp new file mode 100644 index 000000000..8b95ccd96 --- /dev/null +++ b/moses/src/TargetWordInsertionFeature.cpp @@ -0,0 +1,97 @@ +#include <sstream> +#include "TargetWordInsertionFeature.h" +#include "Phrase.h" +#include "TargetPhrase.h" +#include "Hypothesis.h" +#include "ChartHypothesis.h" +#include "ScoreComponentCollection.h" +#include "TranslationOption.h" + +namespace Moses { + +using namespace std; + +bool TargetWordInsertionFeature::Load(const std::string &filePath) +{ + ifstream inFile(filePath.c_str()); + if (!inFile) + { + cerr << "could not open file " << filePath << endl; + return false; + } + + std::string line; + while (getline(inFile, line)) { + m_vocab.insert(line); + } + + inFile.close(); + + m_unrestricted = false; + return true; +} + +void TargetWordInsertionFeature::Evaluate( + const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const +{ + const TargetPhrase& targetPhrase = context.GetTargetPhrase(); + const AlignmentInfo &alignmentInfo = targetPhrase.GetAlignmentInfo(); + const AlignmentInfo::CollType &alignment = alignmentInfo.GetAlignments(); + ComputeFeatures(targetPhrase, accumulator, alignment); +} + +void TargetWordInsertionFeature::EvaluateChart( + const ChartBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const +{ + const TargetPhrase& targetPhrase = context.GetTargetPhrase(); + const AlignmentInfo &alignmentInfo = context.GetTargetPhrase().GetAlignmentInfo(); + const AlignmentInfo::CollType &alignment = alignmentInfo.GetTerminalAlignments(); + ComputeFeatures(targetPhrase, accumulator, alignment); +} + +void TargetWordInsertionFeature::ComputeFeatures(const TargetPhrase& targetPhrase, + ScoreComponentCollection* accumulator, + const AlignmentInfo::CollType &alignment) const +{ + // handle special case: unknown words (they have no word alignment) + size_t targetLength = targetPhrase.GetSize(); + size_t sourceLength = targetPhrase.GetSourcePhrase().GetSize(); + if (targetLength == 1 && sourceLength == 1) { + const Factor* f1 = targetPhrase.GetWord(0).GetFactor(1); + if (f1 && f1->GetString().compare(UNKNOWN_FACTOR) == 0) { + return; + } + } + + // flag aligned words + bool aligned[16]; + CHECK(targetLength < 16); + for(size_t i=0; i<targetLength; i++) { + aligned[i] = false; + } + for (AlignmentInfo::const_iterator alignmentPoint = alignment.begin(); alignmentPoint != alignment.end(); alignmentPoint++) { + aligned[ alignmentPoint->second ] = true; + } + + // process unaligned target words + for(size_t i=0; i<targetLength; i++) { + if (!aligned[i]) { + Word w = targetPhrase.GetWord(i); + if (!w.IsNonTerminal()) { + const string &word = w.GetFactor(m_factorType)->GetString(); + if (word != "<s>" && word != "</s>") { + if (!m_unrestricted && m_vocab.find( word ) == m_vocab.end()) { + accumulator->PlusEquals(this,"OTHER",1); + } + else { + accumulator->PlusEquals(this,word,1); + } + } + } + } + } +} + +} diff --git a/moses/src/TargetWordInsertionFeature.h b/moses/src/TargetWordInsertionFeature.h new file mode 100644 index 000000000..3fd22ec37 --- /dev/null +++ b/moses/src/TargetWordInsertionFeature.h @@ -0,0 +1,49 @@ +#ifndef moses_TargetWordInsertionFeature_h +#define moses_TargetWordInsertionFeature_h + +#include <string> +#include <map> + +#include "FeatureFunction.h" +#include "FactorCollection.h" +#include "AlignmentInfo.h" + +namespace Moses +{ + +/** Sets the features for length of source phrase, target phrase, both. + */ +class TargetWordInsertionFeature : public StatelessFeatureFunction { +private: + std::set<std::string> m_vocab; + FactorType m_factorType; + bool m_unrestricted; + +public: + TargetWordInsertionFeature(FactorType factorType = 0): + StatelessFeatureFunction("twi", ScoreProducer::unlimited), + m_factorType(factorType), + m_unrestricted(true) + { + std::cerr << "Initializing target word insertion feature.." << std::endl; + } + + bool Load(const std::string &filePath); + void Evaluate( const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const; + + void EvaluateChart( const ChartBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const; + + void ComputeFeatures(const TargetPhrase& targetPhrase, + ScoreComponentCollection* accumulator, + const AlignmentInfo::CollType &alignment) const; + + // basic properties + std::string GetScoreProducerWeightShortName(unsigned) const { return "twi"; } + size_t GetNumInputScores() const { return 0; } +}; + +} + +#endif // moses_TargetWordInsertionFeature_h diff --git a/moses/src/TranslationOption.cpp b/moses/src/TranslationOption.cpp index e41e3e1ae..e15cf03c9 100644 --- a/moses/src/TranslationOption.cpp +++ b/moses/src/TranslationOption.cpp @@ -40,18 +40,8 @@ TranslationOption::TranslationOption(const WordsRange &wordsRange , const InputType &inputType) : m_targetPhrase(targetPhrase) , m_sourceWordsRange(wordsRange) -{ - // set score - m_scoreBreakdown.PlusEquals(targetPhrase.GetScoreBreakdown()); - - if (inputType.GetType() == SentenceInput) { - Phrase phrase = inputType.GetSubString(wordsRange); - m_sourcePhrase = new Phrase(phrase); - } else { - // TODO lex reordering with confusion network - m_sourcePhrase = new Phrase(*targetPhrase.GetSourcePhrase()); - } -} + , m_scoreBreakdown(targetPhrase.GetScoreBreakdown()) +{} //TODO this should be a factory function! TranslationOption::TranslationOption(const WordsRange &wordsRange @@ -63,38 +53,16 @@ TranslationOption::TranslationOption(const WordsRange &wordsRange , m_futureScore(0) { if (up) { - const ScoreProducer *scoreProducer = (const ScoreProducer *)up; // not sure why none of the c++ cast works - vector<float> score(1); - score[0] = FloorScore(-numeric_limits<float>::infinity()); - m_scoreBreakdown.Assign(scoreProducer, score); - } - - if (inputType.GetType() == SentenceInput) { - Phrase phrase = inputType.GetSubString(wordsRange); - m_sourcePhrase = new Phrase(phrase); - } else { - // TODO lex reordering with confusion network - m_sourcePhrase = new Phrase(*targetPhrase.GetSourcePhrase()); - //the target phrase from a confusion network/lattice has input scores that we want to keep - m_scoreBreakdown.PlusEquals(targetPhrase.GetScoreBreakdown()); - - } + const ScoreProducer *scoreProducer = (const ScoreProducer *)up; // not sure why none of the c++ cast works + vector<float> score(1); + score[0] = FloorScore(-numeric_limits<float>::infinity()); + m_scoreBreakdown.Assign(scoreProducer, score); + } } -TranslationOption::TranslationOption(const TranslationOption ©) - : m_targetPhrase(copy.m_targetPhrase) -//, m_sourcePhrase(new Phrase(*copy.m_sourcePhrase)) // TODO use when confusion network trans opt for confusion net properly implemented - , m_sourcePhrase( (copy.m_sourcePhrase == NULL) ? new Phrase(ARRAY_SIZE_INCR) : new Phrase(*copy.m_sourcePhrase)) - , m_sourceWordsRange(copy.m_sourceWordsRange) - , m_futureScore(copy.m_futureScore) - , m_scoreBreakdown(copy.m_scoreBreakdown) - , m_cachedScores(copy.m_cachedScores) -{} - TranslationOption::TranslationOption(const TranslationOption ©, const WordsRange &sourceWordsRange) : m_targetPhrase(copy.m_targetPhrase) //, m_sourcePhrase(new Phrase(*copy.m_sourcePhrase)) // TODO use when confusion network trans opt for confusion net properly implemented - , m_sourcePhrase( (copy.m_sourcePhrase == NULL) ? new Phrase(ARRAY_SIZE_INCR) : new Phrase(*copy.m_sourcePhrase)) , m_sourceWordsRange(sourceWordsRange) , m_futureScore(copy.m_futureScore) , m_scoreBreakdown(copy.m_scoreBreakdown) @@ -144,6 +112,7 @@ void TranslationOption::CalcScore(const TranslationSystem* system) allLM.CalcScore(GetTargetPhrase(), retFullScore, ngramScore, oovScore, &m_scoreBreakdown); size_t phraseSize = GetTargetPhrase().GetSize(); + // future score m_futureScore = retFullScore - ngramScore + oovScore + m_scoreBreakdown.InnerProduct(StaticData::Instance().GetAllWeights()) - phraseSize * @@ -164,7 +133,7 @@ ostream& operator<<(ostream& out, const TranslationOption& possibleTranslation) void TranslationOption::CacheScores(const ScoreProducer &producer, const Scores &score) { - m_cachedScores[&producer] = new Scores(score); + m_cachedScores[&producer] = score; } } diff --git a/moses/src/TranslationOption.h b/moses/src/TranslationOption.h index fa4065b80..89ebdc05b 100644 --- a/moses/src/TranslationOption.h +++ b/moses/src/TranslationOption.h @@ -24,6 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include <map> #include <vector> +#include <boost/functional/hash.hpp> #include "WordsBitmap.h" #include "WordsRange.h" #include "Phrase.h" @@ -63,7 +64,6 @@ class TranslationOption protected: TargetPhrase m_targetPhrase; /*< output phrase when using this translation option */ - Phrase *m_sourcePhrase; /*< input phrase translated by this */ const WordsRange m_sourceWordsRange; /*< word position in the input that are covered by this translation option */ float m_futureScore; /*< estimate of total cost when using this translation option, includes language model probabilities */ @@ -73,7 +73,7 @@ protected: //! possible to estimate, it is included here. ScoreComponentCollection m_scoreBreakdown; - typedef std::map<const ScoreProducer *, const Scores *> _ScoreCacheMap; + typedef std::map<const ScoreProducer *, Scores> _ScoreCacheMap; _ScoreCacheMap m_cachedScores; public: @@ -86,17 +86,10 @@ public: , const TargetPhrase &targetPhrase , const InputType &inputType , const UnknownWordPenaltyProducer* uwpProducer); - /** copy constructor */ - TranslationOption(const TranslationOption ©); /** copy constructor, but change words range. used by caching */ TranslationOption(const TranslationOption ©, const WordsRange &sourceWordsRange); - ~TranslationOption() { - delete m_sourcePhrase; - for(_ScoreCacheMap::const_iterator it = m_cachedScores.begin(); it != m_cachedScores.end(); ++it) - delete it->second; - } /** returns true if all feature types in featuresToCheck are compatible between the two phrases */ bool IsCompatible(const Phrase& phrase, const std::vector<FactorType>& featuresToCheck) const; @@ -116,7 +109,7 @@ public: /** returns source phrase */ const Phrase *GetSourcePhrase() const { - return m_sourcePhrase; + return &(m_targetPhrase.GetSourcePhrase()); } /** whether source span overlaps with those of a hypothesis */ @@ -158,7 +151,7 @@ public: if(it == m_cachedScores.end()) return NULL; else - return it->second; + return &(it->second); } /** Calculate future score and n-gram score of this trans option, plus the score breakdowns */ @@ -167,8 +160,26 @@ public: void CacheScores(const ScoreProducer &scoreProducer, const Scores &score); TO_STRING(); + + bool operator== (const TranslationOption &rhs) const + { + return m_sourceWordsRange == rhs.m_sourceWordsRange && + m_targetPhrase == rhs.m_targetPhrase; + } + }; + +//XXX: This doesn't look at the alignment. Is this correct? +inline size_t hash_value(const TranslationOption& translationOption) { + size_t seed = 0; + boost::hash_combine(seed, translationOption.GetTargetPhrase()); + boost::hash_combine(seed, translationOption.GetStartPos()); + boost::hash_combine(seed, translationOption.GetEndPos()); + return seed; +} + + } #endif diff --git a/moses/src/TranslationOptionCollection.cpp b/moses/src/TranslationOptionCollection.cpp index 95ac668b1..5b3c4d94f 100644 --- a/moses/src/TranslationOptionCollection.cpp +++ b/moses/src/TranslationOptionCollection.cpp @@ -201,66 +201,73 @@ void TranslationOptionCollection::ProcessUnknownWord() void TranslationOptionCollection::ProcessOneUnknownWord(const Word &sourceWord,size_t sourcePos, size_t length, const Scores *inputScores) { - // unknown word, add as trans opt - FactorCollection &factorCollection = FactorCollection::Instance(); - - size_t isDigit = 0; - - const Factor *f = sourceWord[0]; // TODO hack. shouldn't know which factor is surface - const string &s = f->GetString(); - bool isEpsilon = (s=="" || s==EPSILON); - if (StaticData::Instance().GetDropUnknown()) { - - - isDigit = s.find_first_of("0123456789"); - if (isDigit == 1) - isDigit = 1; - else - isDigit = 0; - // modify the starting bitmap - } - - Phrase* m_unksrc = new Phrase(1); + // unknown word, add as trans opt + FactorCollection &factorCollection = FactorCollection::Instance(); + + size_t isDigit = 0; + + const Factor *f = sourceWord[0]; // TODO hack. shouldn't know which factor is surface + const string &s = f->GetString(); + bool isEpsilon = (s=="" || s==EPSILON); + if (StaticData::Instance().GetDropUnknown()) + { + + + isDigit = s.find_first_of("0123456789"); + if (isDigit == 1) + isDigit = 1; + else + isDigit = 0; + // modify the starting bitmap + } + + Phrase* m_unksrc = new Phrase(1); m_unksrc->AddWord() = sourceWord; - m_unksrcs.push_back(m_unksrc); - - TranslationOption *transOpt; - TargetPhrase targetPhrase(Output); - targetPhrase.SetSourcePhrase(m_unksrc); - if (inputScores != NULL) { - targetPhrase.SetScore(m_system,*inputScores); - } else { - targetPhrase.SetScore(m_system); - } - - if (!(StaticData::Instance().GetDropUnknown() || isEpsilon) || isDigit) { - // add to dictionary - - Word &targetWord = targetPhrase.AddWord(); - - for (unsigned int currFactor = 0 ; currFactor < MAX_NUM_FACTORS ; currFactor++) { - FactorType factorType = static_cast<FactorType>(currFactor); - - const Factor *sourceFactor = sourceWord[currFactor]; - if (sourceFactor == NULL) - targetWord[factorType] = factorCollection.AddFactor(Output, factorType, UNKNOWN_FACTOR); - else - targetWord[factorType] = factorCollection.AddFactor(Output, factorType, sourceFactor->GetString()); - } - //create a one-to-one alignment between UNKNOWN_FACTOR and its verbatim translation - - targetPhrase.SetAlignmentInfo("0-0"); - - } else { - // drop source word. create blank trans opt + m_unksrcs.push_back(m_unksrc); + + TranslationOption *transOpt; + TargetPhrase targetPhrase(Output); + targetPhrase.SetSourcePhrase(*m_unksrc); + if (inputScores != NULL) { + targetPhrase.SetScore(m_system,*inputScores); + } else { + targetPhrase.SetScore(m_system); + } + + if (!(StaticData::Instance().GetDropUnknown() || isEpsilon) || isDigit) + { + // add to dictionary + + Word &targetWord = targetPhrase.AddWord(); + + for (unsigned int currFactor = 0 ; currFactor < MAX_NUM_FACTORS ; currFactor++) + { + FactorType factorType = static_cast<FactorType>(currFactor); + + const Factor *sourceFactor = sourceWord[currFactor]; + if (sourceFactor == NULL) + targetWord[factorType] = factorCollection.AddFactor(Output, factorType, UNKNOWN_FACTOR); + else + targetWord[factorType] = factorCollection.AddFactor(Output, factorType, sourceFactor->GetString()); + } + //create a one-to-one alignment between UNKNOWN_FACTOR and its verbatim translation + + targetPhrase.SetAlignmentInfo("0-0"); + + } + else + { + // drop source word. create blank trans opt + + //targetPhrase.SetAlignment(); + + } + transOpt = new TranslationOption(WordsRange(sourcePos, sourcePos + length - 1), targetPhrase, m_source + , m_system->GetUnknownWordPenaltyProducer()); + transOpt->CalcScore(m_system); + Add(transOpt); - //targetPhrase.SetAlignment(); - } - transOpt = new TranslationOption(WordsRange(sourcePos, sourcePos + length - 1), targetPhrase, m_source - , m_system->GetUnknownWordPenaltyProducer()); - transOpt->CalcScore(m_system); - Add(transOpt); } /** compute future score matrix in a dynamic programming fashion. @@ -413,6 +420,9 @@ void TranslationOptionCollection::CreateTranslationOptions() // Cached lex reodering costs CacheLexReordering(); + + // stateless feature scores + PreCalculateScores(); } void TranslationOptionCollection::IncorporateDLMScores() { @@ -704,6 +714,51 @@ void TranslationOptionCollection::CacheLexReordering() } } } + +void TranslationOptionCollection::PreCalculateScores() +{ + //Figure out which features need to be precalculated + const vector<const StatelessFeatureFunction*>& sfs = + m_system->GetStatelessFeatureFunctions(); + vector<const StatelessFeatureFunction*> precomputedFeatures; + for (unsigned i = 0; i < sfs.size(); ++i) { + if (sfs[i]->ComputeValueInTranslationOption() && + !sfs[i]->ComputeValueInTranslationTable()) { + precomputedFeatures.push_back(sfs[i]); + } + } + //empty coverage vector + WordsBitmap coverage(m_source.GetSize()); + + //Go through translation options and precompute features + for (size_t i = 0; i < m_collection.size(); ++i) { + for (size_t j = 0; j < m_collection[i].size(); ++j) { + for (size_t k = 0; k < m_collection[i][j].size(); ++k) { + const TranslationOption* toption = m_collection[i][j].Get(k); + ScoreComponentCollection& breakdown = m_precalculatedScores[*toption]; + PhraseBasedFeatureContext context(*toption, m_source); + for (size_t si = 0; si < precomputedFeatures.size(); ++si) { + precomputedFeatures[si]->Evaluate(context, &breakdown); + } + } + } + } +} + +void TranslationOptionCollection::InsertPreCalculatedScores + (const TranslationOption& translationOption, ScoreComponentCollection* scoreBreakdown) + const +{ + boost::unordered_map<TranslationOption,ScoreComponentCollection>::const_iterator scoreIter = + m_precalculatedScores.find(translationOption); + if (scoreIter != m_precalculatedScores.end()) { + scoreBreakdown->PlusEquals(scoreIter->second); + } else { + TRACE_ERR("ERROR: " << translationOption << " missing from precalculation cache" << endl); + assert(0); + } +} + //! list of trans opt for a particular span TranslationOptionList &TranslationOptionCollection::GetTranslationOptionList(size_t startPos, size_t endPos) { diff --git a/moses/src/TranslationOptionCollection.h b/moses/src/TranslationOptionCollection.h index c329d5655..5b123421d 100644 --- a/moses/src/TranslationOptionCollection.h +++ b/moses/src/TranslationOptionCollection.h @@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #define moses_TranslationOptionCollection_h #include <list> +#include <boost/unordered_map.hpp> #include "TypeDef.h" #include "TranslationOption.h" #include "TranslationOptionList.h" @@ -69,6 +70,7 @@ protected: const size_t m_maxNoTransOptPerCoverage; /*< maximum number of translation options per input span */ const float m_translationOptionThreshold; /*< threshold for translation options with regard to best option for input span */ std::vector<Phrase*> m_unksrcs; + boost::unordered_map<TranslationOption,ScoreComponentCollection> m_precalculatedScores; TranslationOptionCollection(const TranslationSystem* system, InputType const& src, size_t maxNoTransOptPerCoverage, @@ -101,6 +103,9 @@ protected: void CacheLexReordering(); + //! Pre-calculate most stateless feature values + void PreCalculateScores(); + public: virtual ~TranslationOptionCollection(); @@ -143,6 +148,10 @@ public: return GetTranslationOptionList(coverage.GetStartPos(), coverage.GetEndPos()); } + //! Access these pre-calculated values + void InsertPreCalculatedScores(const TranslationOption& translationOption, + ScoreComponentCollection* scoreBreakdown) const; + TO_STRING(); }; diff --git a/moses/src/TranslationSystem.cpp b/moses/src/TranslationSystem.cpp index 9e9ea14bd..e9a97bbbc 100644 --- a/moses/src/TranslationSystem.cpp +++ b/moses/src/TranslationSystem.cpp @@ -26,6 +26,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include "DecodeStep.h" #include "DummyScoreProducers.h" #include "GlobalLexicalModel.h" +#include "GlobalLexicalModelUnlimited.h" +#include "WordTranslationFeature.h" +#include "PhrasePairFeature.h" #include "LexicalReordering.h" #include "StaticData.h" #include "TranslationSystem.h" @@ -33,149 +36,146 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA using namespace std; -namespace Moses -{ - -const string TranslationSystem::DEFAULT = "default"; - -TranslationSystem::TranslationSystem(const std::string& id, - const WordPenaltyProducer* wpProducer, - const UnknownWordPenaltyProducer* uwpProducer, - const DistortionScoreProducer* distortionProducer) - : m_id(id), m_wpProducer(wpProducer), m_unknownWpProducer(uwpProducer), m_distortionScoreProducer(distortionProducer) -{ - AddFeatureFunction(wpProducer); - AddFeatureFunction(uwpProducer); - if (distortionProducer) { - AddFeatureFunction(distortionProducer); - } -} - -//Insert core 'big' features -void TranslationSystem::AddLanguageModel(LanguageModel* languageModel) -{ - m_languageModels.Add(languageModel); - AddFeatureFunction(languageModel); -} - - -void TranslationSystem::AddDecodeGraph(DecodeGraph* decodeGraph, size_t backoff) -{ - m_decodeGraphs.push_back(decodeGraph); - m_decodeGraphBackoff.push_back(backoff); -} - - -void TranslationSystem::AddReorderModel(LexicalReordering* reorderModel) -{ - m_reorderingTables.push_back(reorderModel); - AddFeatureFunction(reorderModel); -} - +namespace Moses { + + const string TranslationSystem::DEFAULT = "default"; + + TranslationSystem::TranslationSystem(const std::string& id, + const WordPenaltyProducer* wpProducer, + const UnknownWordPenaltyProducer* uwpProducer, + const DistortionScoreProducer* distortionProducer) + : m_id(id), m_wpProducer(wpProducer), m_unknownWpProducer(uwpProducer), m_distortionScoreProducer(distortionProducer) + { + AddFeatureFunction(wpProducer); + AddFeatureFunction(uwpProducer); + if (distortionProducer) { + AddFeatureFunction(distortionProducer); + } + } + + //Insert core 'big' features + void TranslationSystem::AddLanguageModel(LanguageModel* languageModel) { + m_languageModels.Add(languageModel); + AddFeatureFunction(languageModel); + } -void TranslationSystem::AddGlobalLexicalModel(GlobalLexicalModel* globalLexicalModel) -{ - m_globalLexicalModels.push_back(globalLexicalModel); - AddFeatureFunction(globalLexicalModel); -} + void TranslationSystem::AddDecodeGraph(DecodeGraph* decodeGraph, size_t backoff) { + m_decodeGraphs.push_back(decodeGraph); + m_decodeGraphBackoff.push_back(backoff); + } + void TranslationSystem::AddReorderModel(LexicalReordering* reorderModel) { + m_reorderingTables.push_back(reorderModel); + AddFeatureFunction(reorderModel); + } + void TranslationSystem::AddGlobalLexicalModel(GlobalLexicalModel* globalLexicalModel) { + m_globalLexicalModels.push_back(globalLexicalModel); + AddFeatureFunction(globalLexicalModel); + } + + void TranslationSystem::AddFeatureFunction(const FeatureFunction* ff) { + m_producers.push_back(ff); + + if (ff->IsStateless()) { + m_statelessFFs.push_back(static_cast<const StatelessFeatureFunction*>(ff)); + } else { + m_statefulFFs.push_back(static_cast<const StatefulFeatureFunction*>(ff)); + } + } + void TranslationSystem::AddSparseProducer(const FeatureFunction* ff) { + m_sparseProducers.push_back(ff); + } -void TranslationSystem::AddFeatureFunction(const FeatureFunction* ff) -{ - if (ff->IsStateless()) { - const StatelessFeatureFunction* statelessFF = static_cast<const StatelessFeatureFunction*>(ff); - if (!statelessFF->ComputeValueInTranslationOption()) { - m_statelessFFs.push_back(statelessFF); + void TranslationSystem::ConfigDictionaries() { + for (vector<DecodeGraph*>::const_iterator i = m_decodeGraphs.begin(); + i != m_decodeGraphs.end(); ++i) { + for (DecodeGraph::const_iterator j = (*i)->begin(); j != (*i)->end(); ++j) { + const DecodeStep* step = *j; + PhraseDictionaryFeature* pdict = const_cast<PhraseDictionaryFeature*>(step->GetPhraseDictionaryFeature()); + if (pdict) { + m_phraseDictionaries.push_back(pdict); + AddFeatureFunction(pdict); + const_cast<PhraseDictionaryFeature*>(pdict)->InitDictionary(this); + } + GenerationDictionary* gdict = const_cast<GenerationDictionary*>(step->GetGenerationDictionaryFeature()); + if (gdict) { + m_generationDictionaries.push_back(gdict); + AddFeatureFunction(gdict); + } + } + } } - } else { - m_statefulFFs.push_back(static_cast<const StatefulFeatureFunction*>(ff)); - } -} - -void TranslationSystem::ConfigDictionaries() -{ - for (vector<DecodeGraph*>::const_iterator i = m_decodeGraphs.begin(); i != m_decodeGraphs.end(); ++i) { - for (DecodeGraph::const_iterator j = (*i)->begin(); j != (*i)->end(); ++j) { - const DecodeStep* step = *j; - PhraseDictionaryFeature* pdict = const_cast<PhraseDictionaryFeature*>(step->GetPhraseDictionaryFeature()); - if (pdict) { - m_phraseDictionaries.push_back(pdict); - AddFeatureFunction(pdict); - const_cast<PhraseDictionaryFeature*>(pdict)->InitDictionary(this); + + void TranslationSystem::InitializeBeforeSentenceProcessing(const InputType& source) const { + for (vector<PhraseDictionaryFeature*>::const_iterator i = m_phraseDictionaries.begin(); + i != m_phraseDictionaries.end(); ++i) { + (*i)->InitDictionary(this,source); + } + + for(size_t i=0;i<m_reorderingTables.size();++i) { + m_reorderingTables[i]->InitializeForInput(source); } - GenerationDictionary* gdict = const_cast<GenerationDictionary*>(step->GetGenerationDictionaryFeature()); - if (gdict) { - m_generationDictionaries.push_back(gdict); - AddFeatureFunction(gdict); + for(size_t i=0;i<m_globalLexicalModels.size();++i) { + m_globalLexicalModels[i]->InitializeForInput((Sentence const&)source); } + //for(size_t i=0;i<m_statefulFFs.size();++i) { + //} + for(size_t i=0;i<m_statelessFFs.size();++i) { + if (m_statelessFFs[i]->GetScoreProducerWeightShortName() == "glm") + { + ((GlobalLexicalModelUnlimited*)m_statelessFFs[i])->InitializeForInput((Sentence const&)source); + } + } + + LMList::const_iterator iterLM; + for (iterLM = m_languageModels.begin() ; iterLM != m_languageModels.end() ; ++iterLM) + { + LanguageModel &languageModel = **iterLM; + languageModel.InitializeBeforeSentenceProcessing(); + } + } + + void TranslationSystem::CleanUpAfterSentenceProcessing(const InputType& source) const { + + for(size_t i=0;i<m_phraseDictionaries.size();++i) + { + PhraseDictionaryFeature &phraseDictionaryFeature = *m_phraseDictionaries[i]; + PhraseDictionary* phraseDictionary = const_cast<PhraseDictionary*>(phraseDictionaryFeature.GetDictionary()); + phraseDictionary->CleanUp(source); + + } + + for(size_t i=0;i<m_generationDictionaries.size();++i) + m_generationDictionaries[i]->CleanUp(source); + + //something LMs could do after each sentence + LMList::const_iterator iterLM; + for (iterLM = m_languageModels.begin() ; iterLM != m_languageModels.end() ; ++iterLM) + { + LanguageModel &languageModel = **iterLM; + languageModel.CleanUpAfterSentenceProcessing(source); + } + } + + float TranslationSystem::GetWeightWordPenalty() const { + float weightWP = StaticData::Instance().GetWeight(m_wpProducer); + //VERBOSE(1, "Read weightWP from translation sytem: " << weightWP << std::endl); + return weightWP; + } + + float TranslationSystem::GetWeightUnknownWordPenalty() const { + return StaticData::Instance().GetWeight(m_unknownWpProducer); + } + + float TranslationSystem::GetWeightDistortion() const { + CHECK(m_distortionScoreProducer); + return StaticData::Instance().GetWeight(m_distortionScoreProducer); } - } -} - -void TranslationSystem::InitializeBeforeSentenceProcessing(const InputType& source) const -{ - for (vector<PhraseDictionaryFeature*>::const_iterator i = m_phraseDictionaries.begin(); - i != m_phraseDictionaries.end(); ++i) { - (*i)->InitDictionary(this,source); - } - - for(size_t i=0; i<m_reorderingTables.size(); ++i) { - m_reorderingTables[i]->InitializeForInput(source); - } - for(size_t i=0; i<m_globalLexicalModels.size(); ++i) { - m_globalLexicalModels[i]->InitializeForInput((Sentence const&)source); - } - - LMList::const_iterator iterLM; - for (iterLM = m_languageModels.begin() ; iterLM != m_languageModels.end() ; ++iterLM) { - LanguageModel &languageModel = **iterLM; - languageModel.InitializeBeforeSentenceProcessing(); - } -} - -void TranslationSystem::CleanUpAfterSentenceProcessing(const InputType& source) const -{ - - for(size_t i=0; i<m_phraseDictionaries.size(); ++i) { - PhraseDictionaryFeature &phraseDictionaryFeature = *m_phraseDictionaries[i]; - PhraseDictionary* phraseDictionary = const_cast<PhraseDictionary*>(phraseDictionaryFeature.GetDictionary()); - phraseDictionary->CleanUp(source); - - } - - for(size_t i=0; i<m_generationDictionaries.size(); ++i) - m_generationDictionaries[i]->CleanUp(source); - - //something LMs could do after each sentence - LMList::const_iterator iterLM; - for (iterLM = m_languageModels.begin() ; iterLM != m_languageModels.end() ; ++iterLM) { - LanguageModel &languageModel = **iterLM; - languageModel.CleanUpAfterSentenceProcessing(source); - } -} - -float TranslationSystem::GetWeightWordPenalty() const -{ - //const ScoreComponentCollection weights = StaticData::Instance().GetAllWeights(); - size_t wpIndex = StaticData::Instance().GetScoreIndexManager().GetBeginIndex(m_wpProducer->GetScoreBookkeepingID()); - return StaticData::Instance().GetAllWeights()[wpIndex]; -} - -float TranslationSystem::GetWeightUnknownWordPenalty() const -{ - size_t uwpIndex = StaticData::Instance().GetScoreIndexManager(). - GetBeginIndex(m_unknownWpProducer->GetScoreBookkeepingID()); - return StaticData::Instance().GetAllWeights()[uwpIndex]; -} - -float TranslationSystem::GetWeightDistortion() const -{ - CHECK(m_distortionScoreProducer); - size_t distIndex = StaticData::Instance().GetScoreIndexManager(). - GetBeginIndex(m_distortionScoreProducer->GetScoreBookkeepingID()); - return StaticData::Instance().GetAllWeights()[distIndex]; -} + std::vector<float> TranslationSystem::GetTranslationWeights(size_t index) const { + std::vector<float> weights = StaticData::Instance().GetWeights(GetTranslationScoreProducer(index)); + return weights; + } }; diff --git a/moses/src/TranslationSystem.h b/moses/src/TranslationSystem.h index 3880966f6..f9ac3802a 100644 --- a/moses/src/TranslationSystem.h +++ b/moses/src/TranslationSystem.h @@ -39,109 +39,95 @@ class GenerationDictionary; class WordPenaltyProducer; class DistortionScoreProducer; class UnknownWordPenaltyProducer; +class MetaFeatureProducer; class GlobalLexicalModel; /** * Enables the configuration of multiple translation systems. **/ -class TranslationSystem -{ +class TranslationSystem { + + public: + /** Creates a system with the given id */ + TranslationSystem(const std::string& id, + const WordPenaltyProducer* wpProducer, + const UnknownWordPenaltyProducer* uwpProducer, + const DistortionScoreProducer* distortionProducer); + + //Insert core 'big' features + void AddLanguageModel(LanguageModel* languageModel); + void AddDecodeGraph(DecodeGraph* decodeGraph, size_t backoff); + void AddReorderModel(LexicalReordering* reorderModel); + void AddGlobalLexicalModel(GlobalLexicalModel* globalLexicalModel); + + //Insert non-core feature function + void AddFeatureFunction(const FeatureFunction* featureFunction); + + //Insert sparse producer + void AddSparseProducer(const FeatureFunction* sparseProducer); + + //Called after adding the tables in order to set up the dictionaries + void ConfigDictionaries(); + + + const std::string& GetId() const {return m_id;} + + //Lists of tables relevant to this system. + const std::vector<LexicalReordering*>& GetReorderModels() const {return m_reorderingTables;} + const std::vector<DecodeGraph*>& GetDecodeGraphs() const {return m_decodeGraphs;} + const std::vector<size_t>& GetDecodeGraphBackoff() const {return m_decodeGraphBackoff;} + const LMList& GetLanguageModels() const {return m_languageModels;} + const std::vector<GenerationDictionary*>& GetGenerationDictionaries() const {return m_generationDictionaries;} + const std::vector<PhraseDictionaryFeature*>& GetPhraseDictionaries() const {return m_phraseDictionaries;} + + const std::vector<const StatefulFeatureFunction*>& GetStatefulFeatureFunctions() const {return m_statefulFFs;} + const std::vector<const StatelessFeatureFunction*>& GetStatelessFeatureFunctions() const {return m_statelessFFs;} + const std::vector<const FeatureFunction*>& GetSparseProducers() const {return m_sparseProducers;} + + const WordPenaltyProducer *GetWordPenaltyProducer() const { return m_wpProducer; } + const UnknownWordPenaltyProducer *GetUnknownWordPenaltyProducer() const { return m_unknownWpProducer; } + const DistortionScoreProducer* GetDistortionProducer() const {return m_distortionScoreProducer;} + + const PhraseDictionaryFeature *GetTranslationScoreProducer(size_t index) const { return GetPhraseDictionaries().at(index); } + + float GetWeightWordPenalty() const; + float GetWeightUnknownWordPenalty() const; + float GetWeightDistortion() const; + std::vector<float> GetTranslationWeights(size_t index) const; + + //sentence (and thread) specific initialisationn and cleanup + void InitializeBeforeSentenceProcessing(const InputType& source) const; + void CleanUpAfterSentenceProcessing(const InputType& source) const; + + const std::vector<const ScoreProducer*>& GetFeatureFunctions() const { return m_producers; } + + static const std::string DEFAULT; + + + private: + std::string m_id; + + std::vector<DecodeGraph*> m_decodeGraphs; + std::vector<size_t> m_decodeGraphBackoff; + std::vector<LexicalReordering*> m_reorderingTables; + std::vector<PhraseDictionaryFeature*> m_phraseDictionaries; + std::vector<GenerationDictionary*> m_generationDictionaries; + LMList m_languageModels; + std::vector<GlobalLexicalModel*> m_globalLexicalModels; + + //All stateless FFs, except those that cache scores in T-Option + std::vector<const StatelessFeatureFunction*> m_statelessFFs; + //All statefull FFs + std::vector<const StatefulFeatureFunction*> m_statefulFFs; + //All sparse producers that have an activated global weight + std::vector<const FeatureFunction*> m_sparseProducers; + + const WordPenaltyProducer* m_wpProducer; + const UnknownWordPenaltyProducer* m_unknownWpProducer; + const DistortionScoreProducer* m_distortionScoreProducer; + + std::vector<const ScoreProducer*> m_producers; /**< all the score producers in this run */ -public: - /** Creates a system with the given id */ - TranslationSystem(const std::string& id, - const WordPenaltyProducer* wpProducer, - const UnknownWordPenaltyProducer* uwpProducer, - const DistortionScoreProducer* distortionProducer); - - //Insert core 'big' features - void AddLanguageModel(LanguageModel* languageModel); - void AddDecodeGraph(DecodeGraph* decodeGraph, size_t backoff); - void AddReorderModel(LexicalReordering* reorderModel); - void AddGlobalLexicalModel(GlobalLexicalModel* globalLexicalModel); - - //Insert non-core feature function - void AddFeatureFunction(const FeatureFunction* featureFunction); - - //Called after adding the tables in order to set up the dictionaries - void ConfigDictionaries(); - - - const std::string& GetId() const { - return m_id; - } - - //Lists of tables relevant to this system. - const std::vector<LexicalReordering*>& GetReorderModels() const { - return m_reorderingTables; - } - const std::vector<DecodeGraph*>& GetDecodeGraphs() const { - return m_decodeGraphs; - } - const std::vector<size_t>& GetDecodeGraphBackoff() const { - return m_decodeGraphBackoff; - } - const LMList& GetLanguageModels() const { - return m_languageModels; - } - const std::vector<GenerationDictionary*>& GetGenerationDictionaries() const { - return m_generationDictionaries; - } - const std::vector<PhraseDictionaryFeature*>& GetPhraseDictionaries() const { - return m_phraseDictionaries; - } - - const std::vector<const StatefulFeatureFunction*>& GetStatefulFeatureFunctions() const { - return m_statefulFFs; - } - const std::vector<const StatelessFeatureFunction*>& GetStatelessFeatureFunctions() const { - return m_statelessFFs; - } - - const WordPenaltyProducer *GetWordPenaltyProducer() const { - return m_wpProducer; - } - const UnknownWordPenaltyProducer *GetUnknownWordPenaltyProducer() const { - return m_unknownWpProducer; - } - const DistortionScoreProducer* GetDistortionProducer() const { - return m_distortionScoreProducer; - } - - float GetWeightWordPenalty() const; - float GetWeightUnknownWordPenalty() const; - float GetWeightDistortion() const; - - //sentence (and thread) specific initialisationn and cleanup - void InitializeBeforeSentenceProcessing(const InputType& source) const; - void CleanUpAfterSentenceProcessing(const InputType& source) const; - - - - static const std::string DEFAULT; - - - - -private: - std::string m_id; - - std::vector<DecodeGraph*> m_decodeGraphs; - std::vector<size_t> m_decodeGraphBackoff; - std::vector<LexicalReordering*> m_reorderingTables; - std::vector<PhraseDictionaryFeature*> m_phraseDictionaries; - std::vector<GenerationDictionary*> m_generationDictionaries; - LMList m_languageModels; - std::vector<GlobalLexicalModel*> m_globalLexicalModels; - - //All stateless FFs, except those that cache scores in T-Option - std::vector<const StatelessFeatureFunction*> m_statelessFFs; - //All statefull FFs - std::vector<const StatefulFeatureFunction*> m_statefulFFs; - - const WordPenaltyProducer* m_wpProducer; - const UnknownWordPenaltyProducer* m_unknownWpProducer; - const DistortionScoreProducer* m_distortionScoreProducer; }; diff --git a/moses/src/TrellisPath.h b/moses/src/TrellisPath.h index 9380accd5..d8005435c 100644 --- a/moses/src/TrellisPath.h +++ b/moses/src/TrellisPath.h @@ -59,33 +59,36 @@ protected: void InitScore(); public: - - TrellisPath(); // not implemented - - //! create path OF pure hypo - TrellisPath(const Hypothesis *hypo); - - /** create path from another path, deviate at edgeIndex by using arc instead, - * which may change other hypo back from there - */ - TrellisPath(const TrellisPath ©, size_t edgeIndex, const Hypothesis *arc); - - //! get score for this path throught trellis - inline float GetTotalScore() const { - return m_totalScore; - } - - /** list of each hypo/arcs in path. For anything other than the best hypo, it is not possible just to follow the - * m_prevHypo variable in the hypothesis object - */ - inline const std::vector<const Hypothesis *> &GetEdges() const { - return m_path; - } - - //! create a set of next best paths by wiggling 1 of the node at a time. - void CreateDeviantPaths(TrellisPathCollection &pathColl) const; - - //! create a list of next best paths by wiggling 1 of the node at a time. + TrellisPath(); // not implemented + + //! create path OF pure hypo + TrellisPath(const Hypothesis *hypo); + + /** create path from another path, deviate at edgeIndex by using arc instead, + * which may change other hypo back from there + */ + TrellisPath(const TrellisPath ©, size_t edgeIndex, const Hypothesis *arc); + + //! get score for this path throught trellis + inline float GetTotalScore() const { return m_totalScore; } + + /** list of each hypo/arcs in path. For anything other than the best hypo, it is not possible just to follow the + * m_prevHypo variable in the hypothesis object + */ + inline const std::vector<const Hypothesis *> &GetEdges() const + { + return m_path; + } + + inline size_t GetSize() const + { + return m_path.size(); + } + + //! create a set of next best paths by wiggling 1 of the node at a time. + void CreateDeviantPaths(TrellisPathCollection &pathColl) const; + + //! create a list of next best paths by wiggling 1 of the node at a time. void CreateDeviantPaths(TrellisPathList &pathColl) const; inline const ScoreComponentCollection &GetScoreBreakdown() const { diff --git a/moses/src/TypeDef.h b/moses/src/TypeDef.h index b783108b6..a16f49fc2 100644 --- a/moses/src/TypeDef.h +++ b/moses/src/TypeDef.h @@ -200,7 +200,6 @@ typedef std::vector<std::string> WordAlignments; typedef std::vector<FactorType> FactorList; -typedef std::pair<std::vector<std::string const*>,Scores > StringTgtCand; typedef std::pair<std::vector<std::string const*>,WordAlignments > StringWordAlignmentCand; } diff --git a/moses/src/Util.h b/moses/src/Util.h index 58518d2e3..6f485a623 100644 --- a/moses/src/Util.h +++ b/moses/src/Util.h @@ -344,6 +344,33 @@ double GetUserTime(); // dump SGML parser for <seg> tags std::map<std::string, std::string> ProcessAndStripSGML(std::string &line); +/** + * Returns the first string bounded by the delimiters (default delimiters are " " and "\t")i starting from position first_pos + * and and stores the starting position of the next string (in first_str) + */ +inline std::string GetFirstString(const std::string& str, int& first_pos, const std::string& delimiters = " \t") +{ + + std::string first_str; + // Skip delimiters at beginning. + std::string::size_type lastPos = str.find_first_not_of(delimiters, first_pos); + + // Find first "non-delimiter". + std::string::size_type pos = str.find_first_of(delimiters, lastPos); + + if (std::string::npos != pos || std::string::npos != lastPos){ + + first_str = str.substr(lastPos, pos - lastPos); + + // Skip delimiters. Note the "not_of" + lastPos = str.find_first_not_of(delimiters, pos); + + } + + first_pos = lastPos; + return first_str; +} + template<class T> T log_sum (T log_a, T log_b) { diff --git a/moses/src/Word.cpp b/moses/src/Word.cpp index 045fa01b7..3659fa0e3 100644 --- a/moses/src/Word.cpp +++ b/moses/src/Word.cpp @@ -85,6 +85,15 @@ std::string Word::GetString(const vector<FactorType> factorType,bool endWithBlan return strme.str(); } +std::string Word::GetString(FactorType factorType) const +{ + const Factor *factor = m_factorArray[factorType]; + if (factor != NULL) + return factor->GetString(); + else + return NULL; +} + void Word::CreateFromString(FactorDirection direction , const std::vector<FactorType> &factorOrder , const std::string &str @@ -96,7 +105,8 @@ void Word::CreateFromString(FactorDirection direction const std::string& factorDelimiter = StaticData::Instance().GetFactorDelimiter(); TokenizeMultiCharSeparator(wordVec, str, factorDelimiter); //Tokenize(wordVec, str, "|"); - CHECK(wordVec.size() <= factorOrder.size()); + if (!isNonTerminal) + CHECK(wordVec.size() <= factorOrder.size()); const Factor *factor; for (size_t ind = 0; ind < wordVec.size(); ++ind) { diff --git a/moses/src/Word.h b/moses/src/Word.h index aed7430a8..872b90e13 100644 --- a/moses/src/Word.h +++ b/moses/src/Word.h @@ -26,6 +26,9 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include <iostream> #include <vector> #include <list> + +#include "util/murmur_hash.hh" + #include "TypeDef.h" #include "Factor.h" #include "Util.h" @@ -98,6 +101,7 @@ public: * these debugging functions. */ std::string GetString(const std::vector<FactorType> factorType,bool endWithBlank) const; + std::string GetString(FactorType factorType) const; TO_STRING(); //! transitive comparison of Word objects @@ -134,6 +138,9 @@ public: void CreateUnknownWord(const Word &sourceWord); + inline size_t hash() const { + return util::MurmurHashNative(m_factorArray, MAX_NUM_FACTORS*sizeof(Factor*), m_isNonTerminal); + } }; struct WordComparer { @@ -143,6 +150,11 @@ struct WordComparer { } }; + +inline size_t hash_value(const Word& word) { + return word.hash(); +} + } #endif diff --git a/moses/src/WordTranslationFeature.cpp b/moses/src/WordTranslationFeature.cpp new file mode 100644 index 000000000..ab120caf1 --- /dev/null +++ b/moses/src/WordTranslationFeature.cpp @@ -0,0 +1,439 @@ +#include <sstream> +#include <boost/algorithm/string.hpp> +#include "WordTranslationFeature.h" +#include "Phrase.h" +#include "TargetPhrase.h" +#include "Hypothesis.h" +#include "ChartHypothesis.h" +#include "ScoreComponentCollection.h" +#include "TranslationOption.h" +#include <boost/algorithm/string.hpp> + +namespace Moses { + +using namespace std; + +bool WordTranslationFeature::Load(const std::string &filePathSource, const std::string &filePathTarget) +{ + if (m_domainTrigger) { + // domain trigger terms for each input document + ifstream inFileSource(filePathSource.c_str()); + if (!inFileSource){ + cerr << "could not open file " << filePathSource << endl; + return false; + } + + std::string line; + while (getline(inFileSource, line)) { + std::set<std::string> terms; + vector<string> termVector; + boost::split(termVector, line, boost::is_any_of("\t ")); + for (size_t i=0; i < termVector.size(); ++i) + terms.insert(termVector[i]); + + // add term set for current document + m_vocabDomain.push_back(terms); + } + + inFileSource.close(); + } + else { + // restricted source word vocabulary + ifstream inFileSource(filePathSource.c_str()); + if (!inFileSource) + { + cerr << "could not open file " << filePathSource << endl; + return false; + } + + std::string line; + while (getline(inFileSource, line)) { + m_vocabSource.insert(line); + } + + inFileSource.close(); + + // restricted target word vocabulary + ifstream inFileTarget(filePathTarget.c_str()); + if (!inFileTarget) + { + cerr << "could not open file " << filePathTarget << endl; + return false; + } + + while (getline(inFileTarget, line)) { + m_vocabTarget.insert(line); + } + + inFileTarget.close(); + + m_unrestricted = false; + } + return true; +} + +void WordTranslationFeature::Evaluate + (const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const +{ + const Sentence& input = static_cast<const Sentence&>(context.GetSource()); + const TargetPhrase& targetPhrase = context.GetTargetPhrase(); + const AlignmentInfo &alignment = targetPhrase.GetAlignmentInfo(); + + // process aligned words + for (AlignmentInfo::const_iterator alignmentPoint = alignment.begin(); alignmentPoint != alignment.end(); alignmentPoint++) { + const Phrase& sourcePhrase = targetPhrase.GetSourcePhrase(); + int sourceIndex = alignmentPoint->first; + int targetIndex = alignmentPoint->second; + Word ws = sourcePhrase.GetWord(sourceIndex); + if (m_factorTypeSource == 0 && ws.IsNonTerminal()) continue; + Word wt = targetPhrase.GetWord(targetIndex); + if (m_factorTypeSource == 0 && wt.IsNonTerminal()) continue; + string sourceWord = ws.GetFactor(m_factorTypeSource)->GetString(); + string targetWord = wt.GetFactor(m_factorTypeTarget)->GetString(); + if (m_ignorePunctuation) { + // check if source or target are punctuation + char firstChar = sourceWord.at(0); + CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar ); + if(charIterator != m_punctuationHash.end()) + continue; + firstChar = targetWord.at(0); + charIterator = m_punctuationHash.find( firstChar ); + if(charIterator != m_punctuationHash.end()) + continue; + } + + if (!m_unrestricted) { + if (m_vocabSource.find(sourceWord) == m_vocabSource.end()) + sourceWord = "OTHER"; + if (m_vocabTarget.find(targetWord) == m_vocabTarget.end()) + targetWord = "OTHER"; + } + + if (m_simple) { + // construct feature name + stringstream featureName; + featureName << "wt_"; + featureName << sourceWord; + featureName << "~"; + featureName << targetWord; + accumulator->SparsePlusEquals(featureName.str(), 1); + } + if (m_domainTrigger && !m_sourceContext) { + const bool use_topicid = input.GetUseTopicId(); + const bool use_topicid_prob = input.GetUseTopicIdAndProb(); + if (use_topicid || use_topicid_prob) { + if(use_topicid) { + // use topicid as trigger + const long topicid = input.GetTopicId(); + stringstream feature; + feature << "wt_"; + if (topicid == -1) + feature << "unk"; + else + feature << topicid; + + feature << "_"; + feature << sourceWord; + feature << "~"; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); + } + else { + // use topic probabilities + const vector<string> &topicid_prob = *(input.GetTopicIdAndProb()); + if (atol(topicid_prob[0].c_str()) == -1) { + stringstream feature; + feature << "wt_unk_"; + feature << sourceWord; + feature << "~"; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); + } + else { + for (size_t i=0; i+1 < topicid_prob.size(); i+=2) { + stringstream feature; + feature << "wt_"; + feature << topicid_prob[i]; + feature << "_"; + feature << sourceWord; + feature << "~"; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), atof((topicid_prob[i+1]).c_str())); + } + } + } + } + else { + // range over domain trigger words (keywords) + const long docid = input.GetDocumentId(); + for (set<string>::const_iterator p = m_vocabDomain[docid].begin(); p != m_vocabDomain[docid].end(); ++p) { + string sourceTrigger = *p; + stringstream feature; + feature << "wt_"; + feature << sourceTrigger; + feature << "_"; + feature << sourceWord; + feature << "~"; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); + } + } + } + if (m_sourceContext) { + size_t globalSourceIndex = context.GetTranslationOption().GetStartPos() + sourceIndex; + if (!m_domainTrigger && globalSourceIndex == 0) { + // add <s> trigger feature for source + stringstream feature; + feature << "wt_"; + feature << "<s>,"; + feature << sourceWord; + feature << "~"; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); + } + + // range over source words to get context + for(size_t contextIndex = 0; contextIndex < input.GetSize(); contextIndex++ ) { + if (contextIndex == globalSourceIndex) continue; + string sourceTrigger = input.GetWord(contextIndex).GetFactor(m_factorTypeSource)->GetString(); + if (m_ignorePunctuation) { + // check if trigger is punctuation + char firstChar = sourceTrigger.at(0); + CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar ); + if(charIterator != m_punctuationHash.end()) + continue; + } + + const long docid = input.GetDocumentId(); + bool sourceTriggerExists = false; + if (m_domainTrigger) + sourceTriggerExists = m_vocabDomain[docid].find( sourceTrigger ) != m_vocabDomain[docid].end(); + else if (!m_unrestricted) + sourceTriggerExists = m_vocabSource.find( sourceTrigger ) != m_vocabSource.end(); + + if (m_domainTrigger) { + if (sourceTriggerExists) { + stringstream feature; + feature << "wt_"; + feature << sourceTrigger; + feature << "_"; + feature << sourceWord; + feature << "~"; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); + } + } + else if (m_unrestricted || sourceTriggerExists) { + stringstream feature; + feature << "wt_"; + if (contextIndex < globalSourceIndex) { + feature << sourceTrigger; + feature << ","; + feature << sourceWord; + } + else { + feature << sourceWord; + feature << ","; + feature << sourceTrigger; + } + feature << "~"; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); + } + } + } + if (m_targetContext) { + throw runtime_error("Can't use target words outside current translation option in a stateless feature"); + /* + size_t globalTargetIndex = cur_hypo.GetCurrTargetWordsRange().GetStartPos() + targetIndex; + if (globalTargetIndex == 0) { + // add <s> trigger feature for source + stringstream feature; + feature << "wt_"; + feature << sourceWord; + feature << "~"; + feature << "<s>,"; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); + } + + // range over target words (up to current position) to get context + for(size_t contextIndex = 0; contextIndex < globalTargetIndex; contextIndex++ ) { + string targetTrigger = cur_hypo.GetWord(contextIndex).GetFactor(m_factorTypeTarget)->GetString(); + if (m_ignorePunctuation) { + // check if trigger is punctuation + char firstChar = targetTrigger.at(0); + CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar ); + if(charIterator != m_punctuationHash.end()) + continue; + } + + bool targetTriggerExists = false; + if (!m_unrestricted) + targetTriggerExists = m_vocabTarget.find( targetTrigger ) != m_vocabTarget.end(); + + if (m_unrestricted || targetTriggerExists) { + stringstream feature; + feature << "wt_"; + feature << sourceWord; + feature << "~"; + feature << targetTrigger; + feature << ","; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); + } + }*/ + } + } +} + +void WordTranslationFeature::EvaluateChart( + const ChartBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const +{ + const TargetPhrase& targetPhrase = context.GetTargetPhrase(); + const AlignmentInfo &alignmentInfo = targetPhrase.GetAlignmentInfo(); + const AlignmentInfo::CollType &alignment = alignmentInfo.GetTerminalAlignments(); + + // process aligned words + for (AlignmentInfo::const_iterator alignmentPoint = alignment.begin(); alignmentPoint != alignment.end(); alignmentPoint++) { + const Phrase& sourcePhrase = targetPhrase.GetSourcePhrase(); + int sourceIndex = alignmentPoint->first; + int targetIndex = alignmentPoint->second; + Word ws = sourcePhrase.GetWord(sourceIndex); + if (m_factorTypeSource == 0 && ws.IsNonTerminal()) continue; + Word wt = targetPhrase.GetWord(targetIndex); + if (m_factorTypeSource == 0 && wt.IsNonTerminal()) continue; + string sourceWord = ws.GetFactor(m_factorTypeSource)->GetString(); + string targetWord = wt.GetFactor(m_factorTypeTarget)->GetString(); + if (m_ignorePunctuation) { + // check if source or target are punctuation + char firstChar = sourceWord.at(0); + CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar ); + if(charIterator != m_punctuationHash.end()) + continue; + firstChar = targetWord.at(0); + charIterator = m_punctuationHash.find( firstChar ); + if(charIterator != m_punctuationHash.end()) + continue; + } + + if (!m_unrestricted) { + if (m_vocabSource.find(sourceWord) == m_vocabSource.end()) + sourceWord = "OTHER"; + if (m_vocabTarget.find(targetWord) == m_vocabTarget.end()) + targetWord = "OTHER"; + } + + if (m_simple) { + // construct feature name + stringstream featureName; + featureName << "wt_"; + //featureName << ((sourceExists||m_unrestricted) ? sourceWord : "OTHER"); + featureName << sourceWord; + featureName << "~"; + //featureName << ((targetExists||m_unrestricted) ? targetWord : "OTHER"); + featureName << targetWord; + accumulator->SparsePlusEquals(featureName.str(), 1); + } + /* if (m_sourceContext) { + size_t globalSourceIndex = cur_hypo.GetCurrSourceRange().GetStartPos() + sourceIndex; + if (globalSourceIndex == 0) { + // add <s> trigger feature for source + stringstream feature; + feature << "wt_"; + feature << "<s>,"; + feature << sourceWord; + feature << "~"; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); + cerr << feature.str() << endl; + } + + // range over source words to get context + for(size_t contextIndex = 0; contextIndex < input.GetSize(); contextIndex++ ) { + if (contextIndex == globalSourceIndex) continue; + string sourceTrigger = input.GetWord(contextIndex).GetFactor(m_factorTypeSource)->GetString(); + if (m_ignorePunctuation) { + // check if trigger is punctuation + char firstChar = sourceTrigger.at(0); + CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar ); + if(charIterator != m_punctuationHash.end()) + continue; + } + + bool sourceTriggerExists = false; + if (!m_unrestricted) + sourceTriggerExists = m_vocabSource.find( sourceTrigger ) != m_vocabSource.end(); + + if (m_unrestricted || sourceTriggerExists) { + stringstream feature; + feature << "wt_"; + if (contextIndex < globalSourceIndex) { + feature << sourceTrigger; + feature << ","; + feature << sourceWord; + } + else { + feature << sourceWord; + feature << ","; + feature << sourceTrigger; + } + feature << "~"; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); + cerr << feature.str() << endl; + } + } + }*/ +/* if (m_targetContext) { + size_t globalTargetIndex = 0; // TODO +// size_t globalTargetIndex = cur_hypo.GetCurrTargetWordsRange().GetStartPos() + targetIndex; + if (globalTargetIndex == 0) { + // add <s> trigger feature for source + stringstream feature; + feature << "wt_"; + feature << sourceWord; + feature << "~"; + feature << "<s>,"; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); + cerr << feature.str() << endl; + } + + // range over target words (up to current position) to get context + for(size_t contextIndex = 0; contextIndex < globalTargetIndex; contextIndex++ ) { + Phrase outputPhrase = cur_hypo.GetOutputPhrase(); + string targetTrigger = outputPhrase.GetWord(contextIndex).GetFactor(m_factorTypeTarget)->GetString(); + //string targetTrigger = cur_hypo.GetWord(contextIndex).GetFactor(m_factorTypeTarget)->GetString(); + if (m_ignorePunctuation) { + // check if trigger is punctuation + char firstChar = targetTrigger.at(0); + CharHash::const_iterator charIterator = m_punctuationHash.find( firstChar ); + if(charIterator != m_punctuationHash.end()) + continue; + } + + bool targetTriggerExists = false; + if (!m_unrestricted) + targetTriggerExists = m_vocabTarget.find( targetTrigger ) != m_vocabTarget.end(); + + if (m_unrestricted || targetTriggerExists) { + stringstream feature; + feature << "wt_"; + feature << sourceWord; + feature << "~"; + feature << targetTrigger; + feature << ","; + feature << targetWord; + accumulator->SparsePlusEquals(feature.str(), 1); + cerr << feature.str() << endl; + } + } + }*/ + } + +} + +} diff --git a/moses/src/WordTranslationFeature.h b/moses/src/WordTranslationFeature.h new file mode 100644 index 000000000..7f74ae4e3 --- /dev/null +++ b/moses/src/WordTranslationFeature.h @@ -0,0 +1,94 @@ +#ifndef moses_WordTranslationFeature_h +#define moses_WordTranslationFeature_h + +#include <string> +#include <map> + +#include "FeatureFunction.h" +#include "FactorCollection.h" + +#include "Sentence.h" +#include "FFState.h" + +namespace Moses +{ + +/** Sets the features for word translation + */ +class WordTranslationFeature : public StatelessFeatureFunction { + + typedef std::map< char, short > CharHash; + typedef std::vector< std::set<std::string> > DocumentVector; + +private: + std::set<std::string> m_vocabSource; + std::set<std::string> m_vocabTarget; + DocumentVector m_vocabDomain; + FactorType m_factorTypeSource; + FactorType m_factorTypeTarget; + bool m_unrestricted; + bool m_simple; + bool m_sourceContext; + bool m_targetContext; + bool m_domainTrigger; + float m_sparseProducerWeight; + bool m_ignorePunctuation; + CharHash m_punctuationHash; + +public: + WordTranslationFeature(FactorType factorTypeSource, FactorType factorTypeTarget, + bool simple, bool sourceContext, bool targetContext, bool ignorePunctuation, + bool domainTrigger): + StatelessFeatureFunction("wt", ScoreProducer::unlimited), + m_factorTypeSource(factorTypeSource), + m_factorTypeTarget(factorTypeTarget), + m_unrestricted(true), + m_simple(simple), + m_sourceContext(sourceContext), + m_targetContext(targetContext), + m_domainTrigger(domainTrigger), + m_sparseProducerWeight(1), + m_ignorePunctuation(ignorePunctuation) + { + std::cerr << "Initializing word translation feature.. "; + if (m_simple == 1) std::cerr << "using simple word translations.. "; + if (m_sourceContext == 1) std::cerr << "using source context.. "; + if (m_targetContext == 1) std::cerr << "using target context.. "; + if (m_domainTrigger == 1) std::cerr << "using domain triggers.. "; + + // compile a list of punctuation characters + if (m_ignorePunctuation) { + std::cerr << "ignoring punctuation for triggers.. "; + char punctuation[] = "\"'!?¿·()#_,.:;•&@‑/\\0123456789~="; + for (size_t i=0; i < sizeof(punctuation)-1; ++i) + m_punctuationHash[punctuation[i]] = 1; + } + + std::cerr << "done." << std::endl; + } + + bool Load(const std::string &filePathSource, const std::string &filePathTarget); + + const FFState* EmptyHypothesisState(const InputType &) const { + return new DummyState(); + } + + void Evaluate(const PhraseBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const; + + void EvaluateChart(const ChartBasedFeatureContext& context, + ScoreComponentCollection* accumulator) const; + + // basic properties + std::string GetScoreProducerWeightShortName(unsigned) const { return "wt"; } + size_t GetNumInputScores() const { return 0; } + + bool ComputeValueInTranslationOption() const {return true;} + + void SetSparseProducerWeight(float weight) { m_sparseProducerWeight = weight; } + float GetSparseProducerWeight() const { return m_sparseProducerWeight; } +}; + +} + +#endif // moses_WordTranslationFeature_h diff --git a/moses/src/XmlOption.cpp b/moses/src/XmlOption.cpp index 3e5c20689..f5eee83f9 100644 --- a/moses/src/XmlOption.cpp +++ b/moses/src/XmlOption.cpp @@ -342,6 +342,7 @@ bool ProcessAndStripXMLTags(string &line, vector<XmlOption*> &res, ReorderingCon TargetPhrase targetPhrase(Output); targetPhrase.CreateFromString(outputFactorOrder,altTexts[i],factorDelimiter); targetPhrase.SetScore(scoreValue); + // TODO: targetPhrase.SetSourcePhrase() ? XmlOption *option = new XmlOption(range,targetPhrase); CHECK(option); diff --git a/phrase-extract/consolidate.cpp b/phrase-extract/consolidate.cpp index 82807fd42..43b3f32a1 100644 --- a/phrase-extract/consolidate.cpp +++ b/phrase-extract/consolidate.cpp @@ -308,7 +308,7 @@ void processFiles( char* fileNameDirect, char* fileNameIndirect, char* fileNameC fileConsolidated << " ||| " << itemDirect[3]; // counts, for debugging - fileConsolidated << "||| " << countE << " " << countF; // << " " << countEF; + fileConsolidated << "||| " << countE << " " << countF << " " << countEF; if (outputNTLengths) { diff --git a/phrase-extract/score.cpp b/phrase-extract/score.cpp index 71661b618..f764beef7 100644 --- a/phrase-extract/score.cpp +++ b/phrase-extract/score.cpp @@ -26,6 +26,7 @@ #include <assert.h> #include <cstring> #include <set> +#include <algorithm> #include "SafeGetline.h" #include "tables-core.h" @@ -712,8 +713,10 @@ void outputPhrasePair(const PhraseAlignmentCollection &phrasePair, float totalCo // alignment info for non-terminals if (! inverseFlag) { if (hierarchicalFlag) { - // always output alignment if hiero style, but only for non-terms + // always output alignment if hiero style, but only for non-terms + // (eh: output all alignments, needed for some feature functions) assert(phraseT.size() == bestAlignment.alignedToT.size() + 1); + std::vector<std::string> alignment; for(size_t j = 0; j < phraseT.size() - 1; j++) { if (isNonTerminal(vcbT.getWord( phraseT[j] ))) { if (bestAlignment.alignedToT[ j ].size() != 1) { @@ -722,19 +725,27 @@ void outputPhrasePair(const PhraseAlignmentCollection &phrasePair, float totalCo assert(bestAlignment.alignedToT[ j ].size() == 1); } int sourcePos = *(bestAlignment.alignedToT[ j ].begin()); - phraseTableFile << sourcePos << "-" << j << " "; - } - else if (wordAlignmentFlag) { - const std::set<size_t> &sourceSet = bestAlignment.alignedToT[ j ]; - std::set<size_t>::const_iterator iter; - for (iter = sourceSet.begin(); iter != sourceSet.end(); ++iter) - { - int sourcePos = *iter; - phraseTableFile << sourcePos << "-" << j << " "; - } - } - } - } else if (wordAlignmentFlag) { + //phraseTableFile << sourcePos << "-" << j << " "; + std::stringstream point; + point << sourcePos << "-" << j; + alignment.push_back(point.str()); + } else { + set<size_t>::iterator setIter; + for(setIter = (bestAlignment.alignedToT[j]).begin(); setIter != (bestAlignment.alignedToT[j]).end(); setIter++) { + int sourcePos = *setIter; + //phraseTableFile << sourcePos << "-" << j << " "; + std::stringstream point; + point << sourcePos << "-" << j; + alignment.push_back(point.str()); + } + } + } + // now print all alignments, sorted by source index + sort(alignment.begin(), alignment.end()); + for (size_t i = 0; i < alignment.size(); ++i) { + phraseTableFile << alignment[i] << " "; + } + } else if (wordAlignmentFlag) { // alignment info in pb model for(size_t j=0; j<bestAlignment.alignedToT.size(); j++) { const set< size_t > &aligned = bestAlignment.alignedToT[j]; @@ -745,6 +756,7 @@ void outputPhrasePair(const PhraseAlignmentCollection &phrasePair, float totalCo } } + // counts phraseTableFile << " ||| " << totalCount << " " << count; diff --git a/scripts/analysis/smtgui/filter-phrase-table.pl b/scripts/analysis/smtgui/filter-phrase-table.pl index db51da63d..db51da63d 100644..100755 --- a/scripts/analysis/smtgui/filter-phrase-table.pl +++ b/scripts/analysis/smtgui/filter-phrase-table.pl diff --git a/scripts/ems/experiment.meta b/scripts/ems/experiment.meta index b7ad61235..c0483811a 100644 --- a/scripts/ems/experiment.meta +++ b/scripts/ems/experiment.meta @@ -408,7 +408,7 @@ build-sparse-lexical default-name: model/most-frequent-words template: $moses-script-dir/ems/support/build-sparse-lexical-features.perl IN $input-extension $output-extension OUT "$sparse-lexical-features" create-config - in: reordering-table phrase-translation-table generation-table sparse-lexical domains INTERPOLATED-LM:binlm LM:binlm + in: reordering-table phrase-translation-table generation-table sparse-lexical domains sparse-phrase-translation-table INTERPOLATED-LM:binlm LM:binlm out: config ignore-if: use-hiero rerun-on-change: decoding-steps alignment-factors translation-factors reordering-factors generation-factors lexicalized-reordering training-options script decoding-graph-backoff score-settings additional-ini @@ -472,24 +472,51 @@ input-from-sgm out: raw-input default-name: tuning/input.txt template: $moses-script-dir/ems/support/input-from-sgm.perl < IN > OUT +input-devtest-from-sgm + in: input-devtest-sgm + out: raw-input-devtest + default-name: tuning/input.devtest.txt + ignore-unless: use-mira + template: $moses-script-dir/ems/support/input-from-sgm.perl < IN > OUT tokenize-input in: raw-input out: tokenized-input default-name: tuning/input.tok pass-unless: input-tokenizer template: $input-tokenizer < IN > OUT +tokenize-input-devtest + in: raw-input-devtest + out: tokenized-input-devtest + default-name: tuning/input.devtest.tok + pass-unless: input-tokenizer + ignore-unless: use-mira + template: $input-tokenizer < IN > OUT parse-input in: tokenized-input out: parsed-input default-name: tuning/input.parsed pass-unless: input-parser template: $input-parser < IN > OUT +parse-input-devtest + in: tokenized-input-devtest + out: parsed-input-devtest + default-name: tuning/input.devtest.parsed + pass-unless: input-parser + ignore-unless: use-mira + template: $input-parser < IN > OUT parse-relax-input in: parsed-input out: parse-relaxed-input default-name: tuning/input.parse-relaxed pass-unless: input-parse-relaxer template: $input-parse-relaxer < IN.$input-extension > OUT.$input-extension +parse-relax-input-devtest + in: parsed-input-devtest + out: parse-relaxed-input-devtest + default-name: tuning/input.devtest.parse-relaxed + pass-unless: input-parse-relaxer + ignore-unless: use-mira + template: $input-parse-relaxer < IN.$input-extension > OUT.$input-extension factorize-input in: parse-relaxed-input out: factorized-input @@ -498,32 +525,70 @@ factorize-input pass-unless: TRAINING:input-factors error: can't open error: incompatible number of words in factor +factorize-input-devtest + in: parse-relaxed-input-devtest + out: factorized-input-devtest + default-name: tuning/input.devtest.factorized + rerun-on-change: TRAINING:input-factors + pass-unless: TRAINING:input-factors + ignore-unless: use-mira + error: can't open + error: incompatible number of words in factor lowercase-input in: factorized-input - out: cased-input + out: truecased-input default-name: tuning/input.lc pass-unless: input-lowercaser ignore-if: input-truecaser template: $input-lowercaser < IN > OUT +lowercase-input-devtest + in: factorized-input-devtest + out: truecased-input-devtest + default-name: tuning/input.devtest.lc + pass-unless: input-lowercaser + ignore-unless: use-mira + ignore-if: input-truecaser + template: $input-lowercaser < IN > OUT truecase-input in: factorized-input TRUECASER:truecase-model - out: cased-input + out: truecased-input rerun-on-change: input-truecaser default-name: tuning/input.tc ignore-unless: input-truecaser template: $input-truecaser -model IN1.$input-extension < IN > OUT +truecase-input-devtest + in: factorized-input-devtest TRUECASER:truecase-model + out: truecased-input-devtest + rerun-on-change: input-truecaser + default-name: tuning/input.devtest.tc + ignore-unless: AND input-truecaser use-mira + template: $input-truecaser -model IN1.$input-extension < IN > OUT split-input - in: cased-input SPLITTER:splitter-model + in: truecased-input SPLITTER:splitter-model out: input rerun-on-change: input-splitter default-name: tuning/input.split pass-unless: input-splitter template: $input-splitter -model IN1.$input-extension $input-extension < IN > OUT +split-input-devtest + in: truecased-input-devtest SPLITTER:splitter-model + out: input-devtest + rerun-on-change: input-splitter + default-name: tuning/input.devtest.split + pass-unless: input-splitter + ignore-unless: use-mira + template: $input-splitter -model IN1.$input-extension $input-extension < IN > OUT reference-from-sgm in: reference-sgm input-sgm out: raw-reference default-name: tuning/reference.txt template: $moses-script-dir/ems/support/reference-from-sgm.perl IN IN1 OUT +reference-devtest-from-sgm + in: reference-devtest-sgm input-devtest-sgm + out: raw-reference-devtest + default-name: tuning/reference.devtest.txt + ignore-unless: use-mira + template: $moses-script-dir/ems/support/reference-from-sgm.perl IN IN1 OUT tokenize-reference in: raw-reference out: tokenized-reference @@ -531,29 +596,62 @@ tokenize-reference pass-unless: output-tokenizer multiref: $moses-script-dir/ems/support/run-command-on-multiple-refsets.perl template: $output-tokenizer < IN > OUT +tokenize-reference-devtest + in: raw-reference-devtest + out: tokenized-reference-devtest + default-name: tuning/reference.devtest.tok + pass-unless: output-tokenizer + ignore-unless: use-mira + multiref: $moses-script-dir/ems/support/run-command-on-multiple-refsets.perl + template: $output-tokenizer < IN > OUT lowercase-reference in: tokenized-reference - out: cased-reference + out: truecased-reference default-name: tuning/reference.lc pass-unless: output-lowercaser ignore-if: output-truecaser multiref: $moses-script-dir/ems/support/run-command-on-multiple-refsets.perl template: $output-lowercaser < IN > OUT +lowercase-reference-devtest + in: tokenized-reference-devtest + out: truecased-reference-devtest + default-name: tuning/reference.devtest.lc + pass-unless: output-lowercaser + ignore-if: output-truecaser + ignore-unless: use-mira + multiref: $moses-script-dir/ems/support/run-command-on-multiple-refsets.perl + template: $output-lowercaser < IN > OUT truecase-reference in: tokenized-reference TRUECASER:truecase-model - out: cased-reference + out: truecased-reference rerun-on-change: output-truecaser default-name: tuning/reference.tc ignore-unless: output-truecaser multiref: $moses-script-dir/ems/support/run-command-on-multiple-refsets.perl template: $output-truecaser -model IN1.$output-extension < IN > OUT +truecase-reference-devtest + in: tokenized-reference-devtest TRUECASER:truecase-model + out: truecased-reference-devtest + rerun-on-change: output-truecaser + default-name: tuning/reference.devtest.tc + ignore-unless: AND output-truecaser use-mira + multiref: $moses-script-dir/ems/support/run-command-on-multiple-refsets.perl + template: $output-truecaser -model IN1.$output-extension < IN > OUT split-reference - in: cased-reference SPLITTER:splitter-model + in: truecased-reference SPLITTER:splitter-model out: reference default-name: tuning/reference.split pass-unless: output-splitter multiref: $moses-script-dir/ems/support/run-command-on-multiple-refsets.perl template: $output-splitter -model IN1.$output-extension < IN > OUT +split-reference-devtest + in: truecased-reference-devtest SPLITTER:splitter-model + out: reference-devtest + default-name: tuning/reference.devtest.split + pass-unless: output-splitter + ignore-unless: use-mira + multiref: $moses-script-dir/ems/support/run-command-on-multiple-refsets.perl + template: $output-splitter -model IN1.$output-extension < IN > OUT filter in: input TRAINING:phrase-translation-table TRAINING:reordering-table TRAINING:domains out: filtered-dir @@ -562,6 +660,14 @@ filter pass-if: TRAINING:binarize-all ignore-if: use-hiero error: already exists. Please delete +filter-devtest + in: input-devtest TRAINING:phrase-translation-table TRAINING:reordering-table + out: filtered-dir-devtest + default-name: tuning/filtered.devtest + rerun-on-change: filter-settings + pass-if: TRAINING:binarize-all + ignore-unless: use-mira + error: already exists. Please delete apply-filter in: TRAINING:config filtered-dir out: filtered-config @@ -569,8 +675,15 @@ apply-filter pass-if: TRAINING:binarize-all ignore-if: use-hiero template: $moses-script-dir/ems/support/substitute-filtered-tables.perl IN1/moses.ini < IN > OUT +apply-filter-devtest + in: TRAINING:config filtered-dir-devtest + out: filtered-config-devtest + default-name: tuning/moses.filtered.devtest.ini + pass-if: TRAINING:binarize-all + ignore-unless: use-mira + template: $moses-script-dir/ems/support/substitute-filtered-tables.perl IN1/moses.ini < IN > OUT tune - in: filtered-config input reference + in: filtered-config input reference filtered-config-devtest input-devtest reference-devtest out: weight-config ignore-if: use-hiero qsub-script: yes @@ -640,20 +753,20 @@ factorize-input error: incompatible number of words in factor lowercase-input in: factorized-input - out: cased-input + out: truecased-input default-name: evaluation/input.lc pass-unless: input-lowercaser ignore-if: input-truecaser template: $input-lowercaser < IN > OUT truecase-input in: factorized-input TRUECASER:truecase-model - out: cased-input + out: truecased-input default-name: evaluation/input.tc rerun-on-change: input-truecaser ignore-unless: input-truecaser template: $input-truecaser -model IN1.$input-extension < IN > OUT split-input - in: cased-input SPLITTER:splitter-model + in: truecased-input SPLITTER:splitter-model out: input default-name: evaluation/input.split pass-unless: input-splitter diff --git a/scripts/ems/experiment.perl b/scripts/ems/experiment.perl index 7dd83151d..d5386db75 100755 --- a/scripts/ems/experiment.perl +++ b/scripts/ems/experiment.perl @@ -961,9 +961,15 @@ sub define_step { elsif ($DO_STEP[$i] eq 'TUNING:factorize-input') { &define_tuningevaluation_factorize($i); } + elsif ($DO_STEP[$i] eq 'TUNING:factorize-input-devtest') { + &define_tuningevaluation_factorize($i); + } elsif ($DO_STEP[$i] eq 'TUNING:filter') { &define_tuningevaluation_filter(undef,$i); } + elsif ($DO_STEP[$i] eq 'TUNING:filter-devtest') { + &define_tuningevaluation_filter(undef,$i); + } elsif ($DO_STEP[$i] eq 'TUNING:tune') { &define_tuning_tune($i); } @@ -1034,9 +1040,9 @@ sub execute_steps { delete($DO{$i}); $repeat_if_passed = 1; } - } + } - print "number of steps doable or running: ".(scalar keys %DO)." at ".`date`; + print "number of steps doable or running: ".(scalar keys %DO)." at ".`date`; foreach my $step (keys %DO) { print "\t".($DO{$step}==2?"running: ":"doable: ").$DO_STEP[$step]."\n"; } return unless scalar keys %DO; @@ -1138,8 +1144,6 @@ sub write_info { my $step = $DO_STEP[$i]; my $module_set = $step; $module_set =~ s/:[^:]+$//; - - open(INFO,">".&versionize(&step_file($i)).".INFO") or die "Cannot open: $!"; my %VALUE = &get_parameters_relevant_for_re_use($i); foreach my $parameter (keys %VALUE) { @@ -1566,42 +1570,241 @@ sub factorize_one_language { sub define_tuning_tune { my ($step_id) = @_; my $dir = &check_and_get("GENERAL:working-dir"); - - my ($tuned_config, - $config,$input,$reference) = &get_output_and_input($step_id); my $tuning_script = &check_and_get("TUNING:tuning-script"); - my $scripts = &check_backoff_and_get("TUNING:moses-script-dir"); - my $nbest_size = &check_and_get("TUNING:nbest"); - my $lambda = &backoff_and_get("TUNING:lambda"); - my $tune_continue = &backoff_and_get("TUNING:continue"); - my $tune_inputtype = &backoff_and_get("TUNING:inputtype"); - my $jobs = &backoff_and_get("TUNING:jobs"); - my $decoder = &check_backoff_and_get("TUNING:decoder"); - - my $decoder_settings = &backoff_and_get("TUNING:decoder-settings"); - $decoder_settings = "" unless $decoder_settings; - $decoder_settings .= " -v 0 " unless $CLUSTER && $jobs; - - my $tuning_settings = &backoff_and_get("TUNING:tuning-settings"); - $tuning_settings = "" unless $tuning_settings; - - my $cmd = "$tuning_script $input $reference $decoder $config --nbest $nbest_size --working-dir $dir/tuning/tmp.$VERSION --decoder-flags \"$decoder_settings\" --rootdir $scripts $tuning_settings --no-filter-phrase-table"; - $cmd .= " --lambdas \"$lambda\"" if $lambda; - $cmd .= " --continue" if $tune_continue; - $cmd .= " --inputtype $tune_inputtype" if $tune_inputtype; - - my $qsub_args = &get_qsub_args("TUNING"); - $cmd .= " --queue-flags=\"$qsub_args\"" if ($CLUSTER && $qsub_args); - $cmd .= " --jobs $jobs" if $CLUSTER && $jobs; + my $use_mira = &backoff_and_get("TUNING:use-mira", 0); + + # the last 3 variables are only used for mira tuning + my ($tuned_config,$config,$input,$reference,$config_devtest,$input_devtest,$reference_devtest) = &get_output_and_input($step_id); + + my $cmd = ""; + if ($use_mira) { + my $addTags = &backoff_and_get("TUNING:add-tags"); + my $use_jackknife = &backoff_and_get("TUNING:use-jackknife"); + if ($addTags && !$use_jackknife) { + my $input_with_tags = $input.".".$VERSION.".tags"; + `$addTags < $input > $input_with_tags`; + $input = $input_with_tags; + } + + my $addTagsDevtest = &backoff_and_get("TUNING:add-tags-devtest"); + if ($addTagsDevtest) { + my $input_devtest_with_tags = $input_devtest.".".$VERSION.".tags"; + `$addTagsDevtest < $input_devtest > $input_devtest_with_tags`; + $input_devtest = $input_devtest_with_tags; + } + + my $experiment_dir = "$dir/tuning/tmp.$VERSION"; + system("mkdir -p $experiment_dir"); + + my $mira_config = "$experiment_dir/mira-config.$VERSION."; + my $mira_config_log = $mira_config."log"; + $mira_config .= "cfg"; + + write_mira_config($mira_config,$experiment_dir,$config,$input,$reference,$config_devtest,$input_devtest,$reference_devtest); + #$cmd = "$tuning_script -config $mira_config -exec >& $mira_config_log"; + # we want error messages in top-level log file + $cmd = "$tuning_script -config $mira_config -exec "; + + # write script to select the best set of weights after training for the specified number of epochs --> + # cp to tuning/tmp.?/moses.ini + my $script_filename = "$experiment_dir/selectBestWeights."; + my $script_filename_log = $script_filename."log"; + $script_filename .= "perl"; + my $weight_output_file = "$experiment_dir/moses.ini"; + write_selectBestMiraWeights($experiment_dir, $script_filename, $weight_output_file); + $cmd .= "\n$script_filename >& $script_filename_log"; + } + else { + my $scripts = &check_backoff_and_get("TUNING:moses-script-dir"); + my $nbest_size = &check_and_get("TUNING:nbest"); + my $lambda = &backoff_and_get("TUNING:lambda"); + my $tune_continue = &backoff_and_get("TUNING:continue"); + my $skip_decoder = &backoff_and_get("TUNING:skip-decoder"); + my $tune_inputtype = &backoff_and_get("TUNING:inputtype"); + my $jobs = &backoff_and_get("TUNING:jobs"); + my $decoder = &check_backoff_and_get("TUNING:decoder"); + + my $decoder_settings = &backoff_and_get("TUNING:decoder-settings"); + $decoder_settings = "" unless $decoder_settings; + $decoder_settings .= " -v 0 " unless $CLUSTER && $jobs; + + my $tuning_settings = &backoff_and_get("TUNING:tuning-settings"); + $tuning_settings = "" unless $tuning_settings; + + $cmd = "$tuning_script $input $reference $decoder $config --nbest $nbest_size --working-dir $dir/tuning/tmp.$VERSION --decoder-flags \"$decoder_settings\" --rootdir $scripts $tuning_settings --no-filter-phrase-table"; + $cmd .= " --lambdas \"$lambda\"" if $lambda; + $cmd .= " --continue" if $tune_continue; + $cmd .= " --skip-decoder" if $skip_decoder; + $cmd .= " --inputtype $tune_inputtype" if $tune_inputtype; + + my $qsub_args = &get_qsub_args("TUNING"); + $cmd .= " --queue-flags=\"$qsub_args\"" if ($CLUSTER && $qsub_args); + $cmd .= " --jobs $jobs" if $CLUSTER && $jobs; + my $tuning_dir = $tuned_config; + $tuning_dir =~ s/\/[^\/]+$//; + $cmd .= "\nmkdir -p $tuning_dir"; + } - my $tuning_dir = $tuned_config; - $tuning_dir =~ s/\/[^\/]+$//; - $cmd .= "\nmkdir -p $tuning_dir"; $cmd .= "\ncp $dir/tuning/tmp.$VERSION/moses.ini $tuned_config"; &create_step($step_id,$cmd); } +sub write_mira_config { + my ($config_filename,$expt_dir,$tune_filtered_ini,$input,$reference,$devtest_filtered_ini,$input_devtest,$reference_devtest) = @_; + my $moses_src_dir = &check_and_get("GENERAL:moses-src-dir"); + my $mira_src_dir = &backoff_and_get("GENERAL:mira-src-dir"); + my $tuning_decoder_settings = &check_and_get("TUNING:decoder-settings"); + my $start_weights = &backoff_and_get("TUNING:start-weight-config"); + my $tuning_settings = &check_and_get("TUNING:tuning-settings"); + my $jobs = 10; # this overwrites the default in training-expt.perl + if ($tuning_settings =~ /^(.*)--jobs (\d+)(.*)$/) { + $jobs = $2; + $tuning_settings = $1.$3; + $tuning_settings =~ s/ +/ /; + $tuning_settings =~ s/^ //; + $tuning_settings =~ s/ $//; + } + my $use_jackknife = &backoff_and_get("TUNING:use-jackknife"); + + # are we tuning a meta feature? + my $tune_meta_feature = &backoff_and_get("TUNING:tune-meta-feature"); + + my $tune_filtered_ini_start; + if (!$use_jackknife) { + $tune_filtered_ini =~ /.*\/([A-Za-z0-9\.\-\_]*)$/; + $tune_filtered_ini_start = $1; + $tune_filtered_ini_start = $expt_dir."/".$tune_filtered_ini_start.".start"; + if ($start_weights) { + # apply start weights to filtered ini file, and pass the new ini to mira + print "DEBUG: $RealBin/support/reuse-weights.perl $start_weights < $tune_filtered_ini > $tune_filtered_ini_start \n"; + system("$RealBin/support/reuse-weights.perl $start_weights < $tune_filtered_ini > $tune_filtered_ini_start"); + } + } + + # do we want to continue an interrupted experiment? + my $continue_expt = &backoff_and_get("TUNING:continue-expt"); + my $continue_epoch = &backoff_and_get("TUNING:continue-epoch"); + my $continue_weights = &backoff_and_get("TUNING:continue-weights"); + + # mira config file + open(CFG, ">$config_filename"); + print CFG "[general] \n"; + print CFG "name=expt \n"; + print CFG "fold=0 \n"; + print CFG "mpienv=openmpi_fillup_mark2 \n"; + if ($mira_src_dir) { + print CFG "moses-home=".$mira_src_dir."\n"; + } + else { + print CFG "moses-home=".$moses_src_dir."\n"; + } + print CFG "working-dir=".$expt_dir."\n"; + if ($continue_expt && $continue_expt > 0) { + print CFG "continue-expt=".$continue_expt."\n"; + print CFG "continue-epoch=".$continue_epoch."\n"; + print CFG "continue-weights=".$continue_weights."\n"; + } + print CFG "tune-meta-feature=1 \n" if ($tune_meta_feature); + print CFG "jackknife=1 \n" if ($use_jackknife); + print CFG "wait-for-bleu=1 \n\n"; + #print CFG "decoder-settings=".$tuning_decoder_settings."\n\n"; + print CFG "[train] \n"; + print CFG "trainer=\${moses-home}/bin/mira \n"; + if ($use_jackknife) { + print CFG "input-files-folds="; + for my $i (0..9) { + my $addTags = &backoff_and_get("TUNING:add-tags"); + if ($addTags) { + my $input_with_tags = $input.".".$VERSION.".tags"; + `$addTags.only$i < $input.only$i > $input_with_tags.only$i`; + + print CFG $input_with_tags.".only$i, " if $i<9; + print CFG $input_with_tags.".only$i" if $i==9; + } + else { + print CFG $input.".only$i, " if $i<9; + print CFG $input.".only$i" if $i==9; + } + } + print CFG "\n"; + print CFG "reference-files-folds="; + for my $i (0..9) { + print CFG $reference.".only$i, " if $i<9; + print CFG $reference.".only$i" if $i==9; + } + print CFG "\n"; + print CFG "moses-ini-files-folds="; + for my $i (0..9) { + print CFG $start_weights.".wo$i, " if $i<9; + print CFG $start_weights.".wo$i" if $i==9; + } + print CFG "\n"; + } + else { + print CFG "input-file=".$input."\n"; + print CFG "reference-files=".$reference."\n"; + if ($start_weights) { + print CFG "moses-ini-file=".$tune_filtered_ini_start."\n"; + } + else { + print CFG "moses-ini-file=".$tune_filtered_ini."\n"; + } + } + print CFG "decoder-settings=".$tuning_decoder_settings." -text-type \"dev\"\n"; + print CFG "hours=48 \n"; + print CFG "jobs=$jobs \n"; + print CFG "extra-args=".$tuning_settings."\n\n"; + print CFG "[devtest] \n"; + if (&get("TRAINING:hierarchical-rule-set")) { + print CFG "moses=\${moses-home}/bin/moses_chart \n"; + } + else { + print CFG "moses=\${moses-home}/bin/moses \n"; + } + # use multi-bleu to select the best set of weights + print CFG "bleu=\${moses-home}/scripts/generic/multi-bleu.perl \n"; + print CFG "input-file=".$input_devtest."\n"; + print CFG "reference-file=".$reference_devtest."\n"; + print CFG "moses-ini-file=".$devtest_filtered_ini."\n"; + print CFG "decoder-settings=".$tuning_decoder_settings." -text-type \"devtest\"\n"; + print CFG "hours=12 \nextra-args= \nskip-dev=1 \nskip-devtest=0 \nskip-submit=0 \n"; + close(CFG); +} + +sub write_selectBestMiraWeights { + my ($expt_dir, $script_filename, $weight_out_file) = @_; + open(SCR, ">$script_filename"); + + print SCR "#!/usr/bin/perl -w \nuse strict; \n\n"; + print SCR "my \@devtest_bleu = glob(\"$expt_dir/*_devtest.bleu\"); \# expt_00_0_devtest.bleu \n"; + print SCR "if (scalar(\@devtest_bleu) == 0) { \n"; + print SCR "\tprint STDERR \"ERROR: no bleu files globbed, cannot find best weights.\\n\"; \n"; + print SCR "\texit(1); \n"; + print SCR "} \n\n"; + print SCR "my (\$best_weights, \$best_id); \n"; + print SCR "my \$best_bleu = -1; \n"; + print SCR "my \$best_ratio = 0; \n"; + print SCR "foreach my \$bleu_file (\@devtest_bleu) { \n"; + print SCR "\t\$bleu_file =~ /_([\\d_]+)_devtest.bleu/; \n"; + print SCR "\tmy \$id = \$1; \n"; + print SCR "\topen(BLEU, \$bleu_file); \n"; + print SCR "\tmy \$bleu = <BLEU>; \n"; + print SCR "\t\$bleu =~ /BLEU = ([\\d\\.]+), .*ratio=([\\d\\.]+), /; \n"; + print SCR "\tif (\$1 > \$best_bleu || (\$1 == \$best_bleu && (abs(1-\$2) < abs(1-\$best_ratio)))) { \n"; + print SCR "\t\t\$best_bleu = \$1; \n"; + print SCR "\t\t\$best_ratio = \$2; \n"; + print SCR "\t\t# expt1-devtest.00_0.ini (incl. path to sparse weights) \n"; + print SCR "\t\t(\$best_weights) = glob(\"$expt_dir/*devtest.\$id.ini\"); \n"; + print SCR "\t} \n"; + print SCR "} \n\n"; + print SCR "print STDERR \"Best weights according to BLEU on devtest set: \$best_weights \\n\"; \n"; + print SCR "system(\"cp \$best_weights $weight_out_file\"); \n\n"; + + close(SCR); + system("chmod u+x $script_filename"); +} + sub define_training_prepare_data { my ($step_id) = @_; @@ -1803,7 +2006,8 @@ sub define_training_build_custom_generation { sub define_training_create_config { my ($step_id) = @_; - my ($config,$reordering_table,$phrase_translation_table,$generation_table,$sparse_lexical_features,$domains,@LM) = &get_output_and_input($step_id); + my ($config,$reordering_table,$phrase_translation_table,$generation_table,$sparse_lexical_features,$domains,$sparse_phrase_table,@LM) + = &get_output_and_input($step_id); my $cmd = &get_training_setting(9); @@ -2232,9 +2436,9 @@ sub define_tuningevaluation_filter { my $input_filter; $input_filter = &get("EVALUATION:$set:input-filter") unless $tuning_flag; $input_filter = &get("TUNING:input-filter") if $tuning_flag; + #print "filter: $input_filter \n"; $input_filter = $input unless $input_filter; - - # additional settings + my $settings = &backoff_and_get("EVALUATION:$set:filter-settings") unless $tuning_flag; $settings = &get("TUNING:filter-settings") if $tuning_flag; $settings = "" unless $settings; @@ -2244,22 +2448,22 @@ sub define_tuningevaluation_filter { $settings .= " -Binarizer \"$binarizer\"" if $binarizer; $settings .= " --Hierarchical" if &get("TRAINING:hierarchical-rule-set"); - # get model, and whether suffix array is used. Determines the pt implementation. + # get model, and whether suffix array is used. Determines the pt implementation. my $sa_exec_dir = &get("TRAINING:suffix-array"); - my ($ptImpl, $numFF); - if ($hierarchical) { - if ($sa_exec_dir) { - $ptImpl = 10; # suffix array - $numFF = 7; - } - else { - $ptImpl = 6; # in-mem SCFG - } - } - else { - $ptImpl = 0; # phrase-based - } + my ($ptImpl, $numFF); + if ($hierarchical) { + if ($sa_exec_dir) { + $ptImpl = 10; # suffix array + $numFF = 7; + } + else { + $ptImpl = 6; # in-mem SCFG + } + } + else { + $ptImpl = 0; # phrase-based + } # config file specified? my ($config,$cmd,$delete_config); @@ -2301,23 +2505,26 @@ sub define_tuningevaluation_filter { } # filter command - if ($sa_exec_dir) { - # suffix array - $cmd .= "$scripts/training/wrappers/adam-suffix-array/suffix-array-extract.sh $sa_exec_dir $phrase_translation_table $input_filter $filter_dir \n"; - - my $escaped_filter_dir = $filter_dir; - $escaped_filter_dir =~ s/\//\\\\\//g; - $cmd .= "cat $config | sed s/10\\ 0\\ 0\\ 7.*/10\\ 0\\ 0\\ 7\\ $escaped_filter_dir/g > $filter_dir/moses.ini \n"; - } - else { - # normal phrase table - $cmd .= "$scripts/training/filter-model-given-input.pl"; - $cmd .= " $filter_dir $config $input_filter $settings\n"; - } - + if ($sa_exec_dir) { + # suffix array + $cmd .= "$scripts/training/wrappers/adam-suffix-array/suffix-array-extract.sh $sa_exec_dir $phrase_translation_table $input_filter $filter_dir \n"; + + my $escaped_filter_dir = $filter_dir; + $escaped_filter_dir =~ s/\//\\\\\//g; + $cmd .= "cat $config | sed s/10\\ 0\\ 0\\ 7.*/10\\ 0\\ 0\\ 7\\ $escaped_filter_dir/g > $filter_dir/moses.ini \n"; + } + else { + # normal phrase table + $cmd .= "$scripts/training/filter-model-given-input.pl"; + $cmd .= " $filter_dir $config $input_filter $settings\n"; + } + # clean-up $cmd .= "rm $config" if $delete_config; + # copy moses.ini into specified file location + $cmd .= "\ncp $filter_dir/moses.ini $config\n"; + &create_step($step_id,$cmd); } @@ -2357,6 +2564,13 @@ sub define_evaluation_decode { } } + my $addTags = &backoff_and_get("EVALUATION:$set:add-tags"); + if ($addTags) { + my $input_with_tags = $input.".".$VERSION.".tags"; + `$addTags < $input > $input_with_tags`; + $input = $input_with_tags; + } + # create command my $cmd; my $nbest_size; @@ -2383,6 +2597,7 @@ sub define_evaluation_decode { $cmd = "$decoder $settings -v 0 -f $config < $input > $system_output"; $cmd .= " -n-best-list $system_output.best$nbest_size $nbest" if $nbest; } + $cmd .= " -text-type \"test\""; &create_step($step_id,$cmd); } diff --git a/scripts/ems/support/reuse-weights.perl b/scripts/ems/support/reuse-weights.perl index 1bff4bbd0..71ac3a0b9 100755 --- a/scripts/ems/support/reuse-weights.perl +++ b/scripts/ems/support/reuse-weights.perl @@ -22,7 +22,7 @@ while(<WEIGHT>) { elsif (/^\[weight\-(\S+)\]/) { $current_weight = $1; } - elsif ($current_weight && /^(([\-\d\.]+)([Ee][+-]?[\d]+)?)$/) { + elsif ($current_weight && /^(([\-\d\.]+)([Ee][+-]?[\d]+)?)$/) { push @{$WEIGHT{$current_weight}},$1; } elsif ($weights_file_flag && !/^\[/ && !/^\s*$/) { @@ -37,11 +37,21 @@ close(WEIGHT); my %IGNORE; while(<STDIN>) { - if (/^\[weight\-(\S+)\]/) { + if (/^\[weight-file\]/) { + $weights_file_flag = 1; + } + elsif (/^\[weight\-(\S+)\]/) { $current_weight = $1; if (!defined($WEIGHT{$current_weight})) { - print STDERR "(reuse-weights) WARNING: no weights for weight-$1, deleting\n"; - $current_weight = "xxx"; + if (/^\[weight\-wt/ or /^\[weight\-pp/) { + print $_; + $_ = <STDIN>; + print $_; + } + else { + print STDERR "(reuse-weights) WARNING: no weights for weight-$1, deleting\n"; + $current_weight = "xxx"; + } } else { print $_; @@ -53,6 +63,11 @@ while(<STDIN>) { elsif ($current_weight && /^([\-\d\.]+)([Ee][+-]?[\d]+)?$/) { $IGNORE{$current_weight}++; } + elsif ($weights_file_flag && !/^\[/ && !/^\s*$/) { + $weights_file_flag = 0; + # if weight-file was not defined in weights.ini, take this version + #$weights_file_spec = "\n[weight-file]\n".$_; + } elsif (/^\[/) { $current_weight = ""; print $_; diff --git a/scripts/regression-testing/create_localized_moses_ini.pl b/scripts/regression-testing/create_localized_moses_ini.pl index b102a4ed4..b102a4ed4 100644..100755 --- a/scripts/regression-testing/create_localized_moses_ini.pl +++ b/scripts/regression-testing/create_localized_moses_ini.pl diff --git a/scripts/regression-testing/tests/mert-moses-new-aggregate/filter-stderr b/scripts/regression-testing/tests/mert-moses-new-aggregate/filter-stderr index dc1853ca0..dc1853ca0 100755..100644 --- a/scripts/regression-testing/tests/mert-moses-new-aggregate/filter-stderr +++ b/scripts/regression-testing/tests/mert-moses-new-aggregate/filter-stderr diff --git a/scripts/regression-testing/tests/mert-moses-new-aggregate/filter-stdout b/scripts/regression-testing/tests/mert-moses-new-aggregate/filter-stdout index 1212cc873..1212cc873 100755..100644 --- a/scripts/regression-testing/tests/mert-moses-new-aggregate/filter-stdout +++ b/scripts/regression-testing/tests/mert-moses-new-aggregate/filter-stdout diff --git a/scripts/regression-testing/tests/mert-moses-new-continue/filter-stderr b/scripts/regression-testing/tests/mert-moses-new-continue/filter-stderr index dc1853ca0..dc1853ca0 100755..100644 --- a/scripts/regression-testing/tests/mert-moses-new-continue/filter-stderr +++ b/scripts/regression-testing/tests/mert-moses-new-continue/filter-stderr diff --git a/scripts/regression-testing/tests/mert-moses-new-continue/filter-stdout b/scripts/regression-testing/tests/mert-moses-new-continue/filter-stdout index 1212cc873..1212cc873 100755..100644 --- a/scripts/regression-testing/tests/mert-moses-new-continue/filter-stdout +++ b/scripts/regression-testing/tests/mert-moses-new-continue/filter-stdout diff --git a/scripts/regression-testing/tests/mert-moses-new-nocase/filter-stderr b/scripts/regression-testing/tests/mert-moses-new-nocase/filter-stderr index dc1853ca0..dc1853ca0 100755..100644 --- a/scripts/regression-testing/tests/mert-moses-new-nocase/filter-stderr +++ b/scripts/regression-testing/tests/mert-moses-new-nocase/filter-stderr diff --git a/scripts/regression-testing/tests/mert-moses-new-nocase/filter-stdout b/scripts/regression-testing/tests/mert-moses-new-nocase/filter-stdout index 1212cc873..1212cc873 100755..100644 --- a/scripts/regression-testing/tests/mert-moses-new-nocase/filter-stdout +++ b/scripts/regression-testing/tests/mert-moses-new-nocase/filter-stdout diff --git a/scripts/regression-testing/tests/mert-moses-new/filter-stderr b/scripts/regression-testing/tests/mert-moses-new/filter-stderr index dc1853ca0..dc1853ca0 100755..100644 --- a/scripts/regression-testing/tests/mert-moses-new/filter-stderr +++ b/scripts/regression-testing/tests/mert-moses-new/filter-stderr diff --git a/scripts/regression-testing/tests/mert-moses-new/filter-stdout b/scripts/regression-testing/tests/mert-moses-new/filter-stdout index 1212cc873..1212cc873 100755..100644 --- a/scripts/regression-testing/tests/mert-moses-new/filter-stdout +++ b/scripts/regression-testing/tests/mert-moses-new/filter-stdout diff --git a/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.pl b/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.pl index 6b7c106e6..6b7c106e6 100644..100755 --- a/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.pl +++ b/scripts/share/nonbreaking_prefixes/nonbreaking_prefix.pl diff --git a/scripts/training/corpus-sizes.perl b/scripts/training/corpus-sizes.perl new file mode 100755 index 000000000..f317c5665 --- /dev/null +++ b/scripts/training/corpus-sizes.perl @@ -0,0 +1,16 @@ +#!/usr/bin/perl -w + +# $Id: consolidate-training-data.perl 928 2009-09-02 02:58:01Z philipp $ + +use strict; + +my ($in,$out,@PART) = @ARGV; + +foreach my $part (@PART) { + die("ERROR: no part $part.$in or $part.$out") if (! -e "$part.$in" || ! -e "$part.$out"); + my $in_size = `cat $part.$in | wc -l`; + my $out_size = `cat $part.$out | wc -l`; + die("number of lines don't match: '$part.$in' ($in_size) != '$part.$out' ($out_size)") + if $in_size != $out_size; + print "$in_size"; +} diff --git a/scripts/training/mert-moses.pl b/scripts/training/mert-moses.pl index 24885f034..4f6af126d 100755 --- a/scripts/training/mert-moses.pl +++ b/scripts/training/mert-moses.pl @@ -77,7 +77,7 @@ $SCRIPTS_ROOTDIR = $ENV{"SCRIPTS_ROOTDIR"} if defined($ENV{"SCRIPTS_ROOTDIR"}); # moses.ini file uses FULL names for lambdas, while this training script # internally (and on the command line) uses ABBR names. my @ABBR_FULL_MAP = qw(d=weight-d lm=weight-l tm=weight-t w=weight-w - g=weight-generation lex=weight-lex I=weight-i); + g=weight-generation lex=weight-lex I=weight-i dlm=weight-dlm pp=weight-pp wt=weight-wt pb=weight-pb lex=weight-lex glm=weight-glm); my %ABBR2FULL = map { split /=/, $_, 2 } @ABBR_FULL_MAP; my %FULL2ABBR = map { my ($a, $b) = split /=/, $_, 2; ($b, $a); } @ABBR_FULL_MAP; @@ -785,7 +785,7 @@ while (1) { $cmd .= $file_settings; - + my %sparse_weights; # sparse features my $pro_optimizer_cmd = "$pro_optimizer $megam_default_options run$run.pro.data"; if ($___PAIRWISE_RANKED_OPTIMIZER) { # pro optimization $cmd = "$mert_pro_cmd $seed_settings $pro_file_settings -o run$run.pro.data ; echo 'not used' > $weights_out_file; $pro_optimizer_cmd"; @@ -795,12 +795,29 @@ while (1) { my $pro_cmd = "$mert_pro_cmd $seed_settings $pro_file_settings -o run$run.pro.data ; $pro_optimizer_cmd"; &submit_or_exec($pro_cmd, "run$run.pro.out", "run$run.pro.err"); # ... get results ... - my %dummy; - ($bestpoint, $devbleu) = &get_weights_from_mert("run$run.pro.out", "run$run.pro.err", scalar @{$featlist->{"names"}}, \%dummy); - - open my $pro_fh, '>', "run$run.init.pro" or die "run$run.init.pro: $!"; - print $pro_fh $bestpoint . "\n"; - close $pro_fh; + ($bestpoint,$devbleu) = &get_weights_from_mert("run$run.pro.out","run$run.pro.err",scalar @{$featlist->{"names"}},\%sparse_weights); + # Get the pro outputs ready for mert. Add the weight ranges, + # and a weight and range for the single sparse feature + $cmd =~ s/--ifile (\S+)/--ifile run$run.init.pro/; + open(MERT_START,$1); + open(PRO_START,">run$run.init.pro"); + print PRO_START $bestpoint." 1\n"; + my $mert_line = <MERT_START>; + $mert_line = <MERT_START>; + chomp $mert_line; + print PRO_START $mert_line." 0\n"; + $mert_line = <MERT_START>; + chomp $mert_line; + print PRO_START $mert_line." 1\n"; + close(PRO_START); + + # Write the sparse weights to file so mert can use them + open(SPARSE_WEIGHTS,">run$run.merge-weights"); + foreach my $fname (keys %sparse_weights) { + print SPARSE_WEIGHTS "$fname $sparse_weights{$fname}\n"; + } + close(SPARSE_WEIGHTS); + $cmd = $cmd." --sparse-weights run$run.merge-weights"; # ... and run mert $cmd =~ s/(--ifile \S+)/$1,run$run.init.pro/; @@ -826,8 +843,8 @@ while (1) { print "run $run end at ".`date`; - my %sparse_weights; # sparse features - ($bestpoint, $devbleu) = &get_weights_from_mert("run$run.$mert_outfile", "run$run.$mert_logfile", scalar @{$featlist->{"names"}}, \%sparse_weights); + ($bestpoint,$devbleu) = &get_weights_from_mert("run$run.$mert_outfile","run$run.$mert_logfile",scalar @{$featlist->{"names"}},\%sparse_weights); + my $merge_weight = 0; die "Failed to parse mert.log, missed Best point there." if !defined $bestpoint || !defined $devbleu; @@ -837,6 +854,10 @@ while (1) { # update my cache of lambda values my @newweights = split /\s+/, $bestpoint; + if ($___PRO_STARTING_POINT) { + $merge_weight = pop @newweights; + } + # interpolate with prior's interation weight, if historic-interpolation is specified if ($___HISTORIC_INTERPOLATION>0 && $run>3) { my %historic_sparse_weights; @@ -876,7 +897,11 @@ while (1) { $sparse_weights_file = "run" . ($run + 1) . ".sparse-weights"; open my $sparse_fh, '>', $sparse_weights_file or die "$sparse_weights_file: $!"; foreach my $feature (keys %sparse_weights) { - print $sparse_fh "$feature $sparse_weights{$feature}\n"; + my $sparse_weight = $sparse_weights{$feature}; + if ($___PRO_STARTING_POINT) { + $sparse_weight *= $merge_weight; + } + print $sparse_fh "$feature $sparse_weight\n"; } close $sparse_fh; } diff --git a/scripts/training/train-model.perl b/scripts/training/train-model.perl index d275823fe..05287afee 100755 --- a/scripts/training/train-model.perl +++ b/scripts/training/train-model.perl @@ -39,7 +39,7 @@ my($_EXTERNAL_BINDIR, $_ROOT_DIR, $_CORPUS_DIR, $_GIZA_E2F, $_GIZA_F2E, $_MODEL_ $_CONTINUE,$_MAX_LEXICAL_REORDERING,$_DO_STEPS, @_ADDITIONAL_INI,$_ADDITIONAL_INI_FILE, $_SPARSE_TRANSLATION_TABLE, - $_DICTIONARY, $_EPPEX, $IGNORE); + $_DICTIONARY, $_SPARSE_PHRASE_FEATURES, $_EPPEX, $IGNORE); my $_CORES = 1; my $debug = 0; # debug this script, do not delete any files in debug mode @@ -123,6 +123,7 @@ $_HELP = 1 'memscore:s' => \$_MEMSCORE, 'force-factored-filenames' => \$_FORCE_FACTORED_FILENAMES, 'dictionary=s' => \$_DICTIONARY, + 'sparse-phrase-features' => \$_SPARSE_PHRASE_FEATURES, 'eppex:s' => \$_EPPEX, 'additional-ini=s' => \@_ADDITIONAL_INI, 'additional-ini-file=s' => \$_ADDITIONAL_INI_FILE, diff --git a/util/tokenize_piece.hh b/util/tokenize_piece.hh index c7e1c8633..1209042d8 100644 --- a/util/tokenize_piece.hh +++ b/util/tokenize_piece.hh @@ -50,6 +50,22 @@ class AnyCharacter { return StringPiece(std::find_first_of(in.data(), in.data() + in.size(), chars_.data(), chars_.data() + chars_.size()), 1); } + StringPiece FindLast(const StringPiece &in) const { + return StringPiece(std::find_end(in.data(), in.data() + in.size(), chars_.data(), chars_.data() + chars_.size()), 1); + } + + private: + StringPiece chars_; +}; + +class AnyCharacterLast { + public: + explicit AnyCharacterLast(const StringPiece &chars) : chars_(chars) {} + + StringPiece Find(const StringPiece &in) const { + return StringPiece(std::find_end(in.data(), in.data() + in.size(), chars_.data(), chars_.data() + chars_.size()), 1); + } + private: StringPiece chars_; }; |