diff options
author | hieuhoang1972 <hieuhoang1972@1f5c12ca-751b-0410-a591-d2e778427230> | 2008-01-29 22:28:13 +0300 |
---|---|---|
committer | hieuhoang1972 <hieuhoang1972@1f5c12ca-751b-0410-a591-d2e778427230> | 2008-01-29 22:28:13 +0300 |
commit | fd60fe93b9e1bb2f210fbea861d78d7a84b37624 (patch) | |
tree | d9cdbd6c1cc4b788bfcfc6f49f634aff8693bae3 /moses | |
parent | 060d26cbb309afe517a4685aeb0726022a405270 (diff) |
roll back last change. seg fault in release build
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@1549 1f5c12ca-751b-0410-a591-d2e778427230
Diffstat (limited to 'moses')
-rwxr-xr-x | moses/src/Hypothesis.cpp | 6 | ||||
-rwxr-xr-x | moses/src/Manager.cpp | 22 | ||||
-rwxr-xr-x | moses/src/Sentence.cpp | 27 | ||||
-rwxr-xr-x | moses/src/Sentence.h | 3 | ||||
-rwxr-xr-x | moses/src/TranslationOption.h | 14 | ||||
-rwxr-xr-x | moses/src/WordsBitmap.cpp | 20 | ||||
-rwxr-xr-x | moses/src/WordsBitmap.h | 6 | ||||
-rw-r--r-- | moses/src/XmlOption.cpp | 44 | ||||
-rw-r--r-- | moses/src/XmlOption.h | 4 |
9 files changed, 34 insertions, 112 deletions
diff --git a/moses/src/Hypothesis.cpp b/moses/src/Hypothesis.cpp index 476af95a8..c5416e939 100755 --- a/moses/src/Hypothesis.cpp +++ b/moses/src/Hypothesis.cpp @@ -185,8 +185,10 @@ int Hypothesis::NGramCompare(const Hypothesis &compare) const // 0 = this ==compare if (m_languageModelStates < compare.m_languageModelStates) return -1; if (m_languageModelStates > compare.m_languageModelStates) return 1; - if (m_sourceCompleted.GetCompressedRepresentation() < compare.m_sourceCompleted.GetCompressedRepresentation()) return -1; - if (m_sourceCompleted.GetCompressedRepresentation() > compare.m_sourceCompleted.GetCompressedRepresentation()) return 1; + + int compareBitmap = m_sourceCompleted.Compare(compare.m_sourceCompleted); + if (compareBitmap != 0) + return compareBitmap; if (m_currSourceWordsRange.GetEndPos() < compare.m_currSourceWordsRange.GetEndPos()) return -1; if (m_currSourceWordsRange.GetEndPos() > compare.m_currSourceWordsRange.GetEndPos()) return 1; if (! StaticData::Instance().GetSourceStartPosMattersForRecombination()) return 0; diff --git a/moses/src/Manager.cpp b/moses/src/Manager.cpp index f077f5173..8b9ef579e 100755 --- a/moses/src/Manager.cpp +++ b/moses/src/Manager.cpp @@ -147,10 +147,10 @@ void Manager::ProcessOneHypothesis(const Hypothesis &hypothesis) for (size_t startPos = hypoFirstGapPos ; startPos < sourceSize ; ++startPos) { - size_t maxSize = sourceSize - startPos; - size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength(); - maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase; - + size_t maxSize = sourceSize - startPos; + size_t maxSizePhrase = StaticData::Instance().GetMaxPhraseLength(); + maxSize = (maxSize < maxSizePhrase) ? maxSize : maxSizePhrase; + for (size_t endPos = startPos ; endPos < startPos + maxSize ; ++endPos) { if (!hypoBitmap.Overlap(WordsRange(startPos, endPos))) @@ -296,20 +296,6 @@ void Manager::ExpandHypothesis(const Hypothesis &hypothesis, const TranslationOp if (debug2) { std::cerr << "::EXT: " << transOpt << "\n"; } #endif Hypothesis *newHypo = hypothesis.CreateNext(transOpt); - // expand hypothesis further if transOpt was linked - for (std::vector<TranslationOption*>::const_iterator iterLinked = transOpt.GetLinkedTransOpts().begin(); - iterLinked != transOpt.GetLinkedTransOpts().end(); iterLinked++) { - const WordsBitmap hypoBitmap = newHypo->GetWordsBitmap(); - if (hypoBitmap.Overlap((**iterLinked).GetSourceWordsRange())) { - // don't want to add a hypothesis that has some but not all of a linked TO set, so return - return; - } - else - { - newHypo->CalcScore(m_transOptColl->GetFutureScore()); - newHypo = newHypo->CreateNext(**iterLinked); - } - } newHypo->CalcScore(m_transOptColl->GetFutureScore()); // logging for the curious diff --git a/moses/src/Sentence.cpp b/moses/src/Sentence.cpp index 5a2f7b2be..a8a720ebc 100755 --- a/moses/src/Sentence.cpp +++ b/moses/src/Sentence.cpp @@ -42,7 +42,7 @@ int Sentence::Read(std::istream& in,const std::vector<FactorType>& factorOrder) //parse XML markup in translation line const StaticData &staticData = StaticData::Instance(); if (staticData.GetXmlInputType() != XmlPassThrough) - m_xmlOptionsList = ProcessAndStripXMLTags(line, *this); + m_xmlOptionsList = ProcessAndStripXMLTags(line); Phrase::CreateFromString(factorOrder, line, factorDelimiter); //only fill the vector if we are parsing XML @@ -50,10 +50,9 @@ int Sentence::Read(std::istream& in,const std::vector<FactorType>& factorOrder) for (size_t i=0; i<GetSize();i++) { m_xmlCoverageMap.push_back(false); } - for (std::vector<TranslationOption*>::const_iterator iterXMLOpts = m_xmlOptionsList.begin(); - iterXMLOpts != m_xmlOptionsList.end(); iterXMLOpts++) { + for (size_t i=0; i< m_xmlOptionsList.size();i++) { //m_xmlOptionsList will be empty for XmlIgnore - for(size_t j=(**iterXMLOpts).GetSourceWordsRange().GetStartPos();j<=(**iterXMLOpts).GetSourceWordsRange().GetEndPos();j++) { + for(size_t j=m_xmlOptionsList[i].startPos;j<=m_xmlOptionsList[i].endPos;j++) { m_xmlCoverageMap[j]=true; } @@ -89,12 +88,24 @@ bool Sentence::XmlOverlap(size_t startPos, size_t endPos) const { void Sentence::GetXmlTranslationOptions(std::vector <TranslationOption*> &list, size_t startPos, size_t endPos) const { //iterate over XmlOptions list, find exact source/target matches + //we don't worry about creating the objects ahead of time because this should only be called once for each unique start/end when a given sentence is processed const std::vector<FactorType> &outputFactorOrder = StaticData::Instance().GetOutputFactorOrder(); - for (std::vector<TranslationOption*>::const_iterator iterXMLOpts = m_xmlOptionsList.begin(); - iterXMLOpts != m_xmlOptionsList.end(); iterXMLOpts++) { - if (startPos == (**iterXMLOpts).GetSourceWordsRange().GetStartPos() && endPos == (**iterXMLOpts).GetSourceWordsRange().GetEndPos()) { - list.push_back(*iterXMLOpts); + for(size_t i=0;i<m_xmlOptionsList.size();i++) { + if (startPos == m_xmlOptionsList[i].startPos && endPos == m_xmlOptionsList[i].endPos) { + //create TranslationOptions + + for (size_t j=0;j<m_xmlOptionsList[i].targetPhrases.size();j++) { + TargetPhrase targetPhrase(Output); + targetPhrase.CreateFromString(outputFactorOrder,m_xmlOptionsList[i].targetPhrases[j],StaticData::Instance().GetFactorDelimiter()); + targetPhrase.SetScore(m_xmlOptionsList[i].targetScores[j]); + WordsRange range(m_xmlOptionsList[i].startPos,m_xmlOptionsList[i].endPos); + + TranslationOption *option = new TranslationOption(range,targetPhrase,*this); + assert(option); + list.push_back(option); + + } } } } diff --git a/moses/src/Sentence.h b/moses/src/Sentence.h index 0f0b4aa7a..8a05787ee 100755 --- a/moses/src/Sentence.h +++ b/moses/src/Sentence.h @@ -48,7 +48,7 @@ class Sentence : public Phrase, public InputType * and returns the value of that tag if present, empty string otherwise */ static std::string ParseXmlTagAttribute(const std::string& tag,const std::string& attributeName); - std::vector <TranslationOption*> m_xmlOptionsList; + std::vector <XmlOption> m_xmlOptionsList; std::vector <bool> m_xmlCoverageMap; public: @@ -90,4 +90,3 @@ class Sentence : public Phrase, public InputType TranslationOptionCollection* CreateTranslationOptionCollection() const; }; - diff --git a/moses/src/TranslationOption.h b/moses/src/TranslationOption.h index 440d4461d..441cd403c 100755 --- a/moses/src/TranslationOption.h +++ b/moses/src/TranslationOption.h @@ -60,7 +60,6 @@ protected: Phrase *m_sourcePhrase; /*< input phrase translated by this */ const WordsRange m_sourceWordsRange; /*< word position in the input that are covered by this translation option */ float m_futureScore; /*< estimate of total cost when using this translation option, includes language model probabilities */ - std::vector<TranslationOption*> m_linkedTransOpts; /* list of linked TOs which must be included with this in any hypothesis */ //! in TranslationOption, m_scoreBreakdown is not complete. It cannot, //! for example, know the full n-gram score since the length of the @@ -113,18 +112,6 @@ public: { return m_sourcePhrase; } - - /** returns linked TOs */ - inline const std::vector<TranslationOption*> &GetLinkedTransOpts() const - { - return m_linkedTransOpts; - } - - /** add link to another TO */ - inline void AddLinkedTransOpt(TranslationOption* to) - { - m_linkedTransOpts.push_back(to); - } /** whether source span overlaps with those of a hypothesis */ bool Overlap(const Hypothesis &hypothesis) const; @@ -180,4 +167,3 @@ public: }; - diff --git a/moses/src/WordsBitmap.cpp b/moses/src/WordsBitmap.cpp index 97c50cda0..316bd9e08 100755 --- a/moses/src/WordsBitmap.cpp +++ b/moses/src/WordsBitmap.cpp @@ -56,23 +56,3 @@ int WordsBitmap::GetFutureCosts(int lastPos) const return sum; } - -std::vector<size_t> WordsBitmap::GetCompressedRepresentation() const -{ - int vectorSize = 1 + m_size / (sizeof(int) * CHAR_BIT); - std::vector<size_t> res(vectorSize, 0); - size_t x=0; size_t ci=0; - for(size_t i=0;i<m_size;++i) { - x <<= 1; - x |= (size_t)m_bitmap[i]; - if ((i + 1) % (sizeof(int) * CHAR_BIT) == 0) { - res[ci++] = x; x = 0; - } - } - x <<= vectorSize * sizeof(int) * CHAR_BIT - m_size; - res[ci] = x; - return res; -} - - - diff --git a/moses/src/WordsBitmap.h b/moses/src/WordsBitmap.h index 7a874d649..77418808d 100755 --- a/moses/src/WordsBitmap.h +++ b/moses/src/WordsBitmap.h @@ -149,11 +149,7 @@ public: { return m_size; } - /** represent this bitmap as 1 or more vector of integers. - * Used for exact matching of source words translated in hypothesis recombination - */ - std::vector<size_t> GetCompressedRepresentation() const; - + //! transitive comparison of WordsBitmap inline int Compare (const WordsBitmap &compare) const { diff --git a/moses/src/XmlOption.cpp b/moses/src/XmlOption.cpp index 6c18aac12..e71dfc97f 100644 --- a/moses/src/XmlOption.cpp +++ b/moses/src/XmlOption.cpp @@ -26,7 +26,6 @@ #include <iostream> #include "Util.h" #include "StaticData.h" -#include "TranslationOption.h" namespace { @@ -92,11 +91,10 @@ inline std::vector<std::string> TokenizeXml(const std::string& str) } -std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const InputType &source) { +std::vector<XmlOption> ProcessAndStripXMLTags(std::string& line) { //parse XML markup in translation line - std::vector<TranslationOption*> res; + std::vector<XmlOption> res; std::string rstr; - std::string linkedStr; if (line.find_first_of('<') == std::string::npos) { return res; } std::vector<std::string> xmlTokens = TokenizeXml(line); std::string tagName = ""; @@ -108,7 +106,6 @@ std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const size_t curWord=0; int numUnary = 0; bool doClose = false; - bool isLinked = false; for (size_t xmlTokenPos = 0 ; xmlTokenPos < xmlTokens.size() ; xmlTokenPos++) { if(!isXmlTag(xmlTokens[xmlTokenPos])) @@ -130,30 +127,7 @@ std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const nextTagName = tag.substr(0,endOfName); tagContents = tag.substr(endOfName+1); } - if (nextTagName == "linked") { - isLinked = true; - linkedStr = ""; - } - else if (nextTagName == "/linked") { - isLinked = false; - // recurse to process linked tags - std::vector<TranslationOption*> tOptions = ProcessAndStripXMLTags(linkedStr, source); - // link them together - std::vector<TranslationOption*>::const_iterator iterTransOpts1; - std::vector<TranslationOption*>::const_iterator iterTransOpts2; - for (iterTransOpts1 = tOptions.begin(); iterTransOpts1 != tOptions.end(); iterTransOpts1++) { - for (iterTransOpts2 = tOptions.begin(); iterTransOpts2 != tOptions.end(); iterTransOpts2++) { - if (iterTransOpts1 != iterTransOpts2) { - (**iterTransOpts1).AddLinkedTransOpt(*iterTransOpts2); - } - } - res.push_back(*iterTransOpts1); - } - } - else if (isLinked) { - linkedStr += xmlTokens[xmlTokenPos]; - } - else if (isOpen) + if (isOpen) { //this is an open tag tagName = nextTagName; @@ -208,7 +182,6 @@ std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const //TODO: deal with multiple XML options here if (StaticData::Instance().GetXmlInputType() != XmlIgnore) { - const std::vector<FactorType> &outputFactorOrder = StaticData::Instance().GetOutputFactorOrder(); for (size_t i=0; i<altTexts.size(); ++i) { //only store options if we aren't ignoring them //set default probability @@ -216,15 +189,7 @@ std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const if (altProbs[i] != "") probValue = Scan<float>(altProbs[i]); //Convert from prob to log-prob float scoreValue = FloorScore(TransformScore(probValue)); - - TargetPhrase targetPhrase(Output); - targetPhrase.CreateFromString(outputFactorOrder,altTexts[i],StaticData::Instance().GetFactorDelimiter()); - targetPhrase.SetScore(scoreValue); - WordsRange range(tagStart,tagEnd); - - TranslationOption *option = new TranslationOption(range,targetPhrase,source); - assert(option); - + XmlOption option(tagStart,tagEnd,altTexts[i],scoreValue); res.push_back(option); } } @@ -240,4 +205,3 @@ std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const return res; } - diff --git a/moses/src/XmlOption.h b/moses/src/XmlOption.h index dc9efe650..e211a6398 100644 --- a/moses/src/XmlOption.h +++ b/moses/src/XmlOption.h @@ -2,7 +2,6 @@ #include <vector> #include <string> -#include "InputType.h" /** This struct is used for storing XML force translation data for a given range in the sentence */ @@ -19,6 +18,5 @@ struct XmlOption { }; -std::vector<TranslationOption*> ProcessAndStripXMLTags(std::string& line, const InputType &source); - +std::vector<XmlOption> ProcessAndStripXMLTags(std::string& line); |