diff options
author | Matthias Huck <huck@i6.informatik.rwth-aachen.de> | 2014-06-12 00:08:22 +0400 |
---|---|---|
committer | Matthias Huck <huck@i6.informatik.rwth-aachen.de> | 2014-06-12 00:08:22 +0400 |
commit | 9a7e568760a467a2dca5c1a128ea9b361d0d73fe (patch) | |
tree | 7ef65ec97fb5067de35f86ba1bfcb128418f25b0 /moses/PP | |
parent | 7380a83f752e2a3c0165f6175b7ac097ac7bd041 (diff) |
SourceLabelsPhraseProperty
Diffstat (limited to 'moses/PP')
-rw-r--r-- | moses/PP/Factory.cpp | 2 | ||||
-rw-r--r-- | moses/PP/SourceLabelsPhraseProperty.cpp | 125 | ||||
-rw-r--r-- | moses/PP/SourceLabelsPhraseProperty.h | 71 |
3 files changed, 198 insertions, 0 deletions
diff --git a/moses/PP/Factory.cpp b/moses/PP/Factory.cpp index a7e06b392..0cc891a82 100644 --- a/moses/PP/Factory.cpp +++ b/moses/PP/Factory.cpp @@ -5,6 +5,7 @@ #include <vector> #include "moses/PP/CountsPhraseProperty.h" +#include "moses/PP/SourceLabelsPhraseProperty.h" #include "moses/PP/TreeStructurePhraseProperty.h" namespace Moses @@ -52,6 +53,7 @@ PhrasePropertyFactory::PhrasePropertyFactory() #define MOSES_PNAME2(name, type) Add(name, new DefaultPhrasePropertyCreator< type >()); MOSES_PNAME2("Counts", CountsPhraseProperty); + MOSES_PNAME2("SourceLabels", SourceLabelsPhraseProperty); MOSES_PNAME2("Tree",TreeStructurePhraseProperty); } diff --git a/moses/PP/SourceLabelsPhraseProperty.cpp b/moses/PP/SourceLabelsPhraseProperty.cpp new file mode 100644 index 000000000..6f18480c5 --- /dev/null +++ b/moses/PP/SourceLabelsPhraseProperty.cpp @@ -0,0 +1,125 @@ +#include "moses/PP/SourceLabelsPhraseProperty.h" +#include <iostream> +#include <cstdio> +#include <cstdlib> +#include <sstream> +#include <string> +#include <queue> +#include <assert.h> +#include <limits> +#include "util/exception.hh" + +namespace Moses +{ + +void SourceLabelsPhraseProperty::ProcessValue() +{ + std::istringstream tokenizer(m_value); + + if (! (tokenizer >> m_nNTs)) { // first token: number of non-terminals (incl. left-hand side) + UTIL_THROW2("SourceLabelsPhraseProperty: Not able to read number of non-terminals. Flawed property?"); + } + assert( m_nNTs > 0 ); + + if (! (tokenizer >> m_totalCount)) { // second token: overall rule count + UTIL_THROW2("SourceLabelsPhraseProperty: Not able to read overall rule count. Flawed property?"); + } + assert( m_totalCount > 0.0 ); + + + + // read source-labelled rule items + + std::priority_queue<float> ruleLabelledCountsPQ; + + while (tokenizer.peek() != EOF) { + try { + + SourceLabelsPhrasePropertyItem item; + size_t numberOfLHSsGivenRHS = std::numeric_limits<std::size_t>::max(); + + if (m_nNTs == 1) { + + item.m_sourceLabelsRHSCount = m_totalCount; + + } else { // rule has right-hand side non-terminals, i.e. it's a hierarchical rule + + for (size_t i=0; i<m_nNTs-1; ++i) { // RHS source non-terminal labels + size_t sourceLabelRHS; + if (! (tokenizer >> sourceLabelRHS) ) { // RHS source non-terminal label + UTIL_THROW2("SourceLabelsPhraseProperty: Not able to read right-hand side label index. Flawed property?"); + } + item.m_sourceLabelsRHS.push_back(sourceLabelRHS); + } + + if (! (tokenizer >> item.m_sourceLabelsRHSCount)) { + UTIL_THROW2("SourceLabelsPhraseProperty: Not able to read right-hand side count. Flawed property?"); + } + + if (! (tokenizer >> numberOfLHSsGivenRHS)) { + UTIL_THROW2("SourceLabelsPhraseProperty: Not able to read number of left-hand sides. Flawed property?"); + } + } + + for (size_t i=0; i<numberOfLHSsGivenRHS && tokenizer.peek()!=EOF; ++i) { // LHS source non-terminal labels seen with this RHS + size_t sourceLabelLHS; + if (! (tokenizer >> sourceLabelLHS)) { // LHS source non-terminal label + UTIL_THROW2("SourceLabelsPhraseProperty: Not able to read left-hand side label index. Flawed property?"); + } + float ruleSourceLabelledCount; + if (! (tokenizer >> ruleSourceLabelledCount)) { + UTIL_THROW2("SourceLabelsPhraseProperty: Not able to read count. Flawed property?"); + } + item.m_sourceLabelsLHSList.push_back( std::make_pair(sourceLabelLHS,ruleSourceLabelledCount) ); + ruleLabelledCountsPQ.push(ruleSourceLabelledCount); + } + + m_sourceLabelItems.push_back(item); + + } catch (const std::exception &e) { + UTIL_THROW2("SourceLabelsPhraseProperty: Read error. Flawed property?"); + } + } + + // keep only top N label vectors + const size_t N=50; + + if (ruleLabelledCountsPQ.size() > N) { + + float topNRuleLabelledCount = std::numeric_limits<int>::max(); + for (size_t i=0; !ruleLabelledCountsPQ.empty() && i<N; ++i) { + topNRuleLabelledCount = ruleLabelledCountsPQ.top(); + ruleLabelledCountsPQ.pop(); + } + + size_t nKept=0; + std::list<SourceLabelsPhrasePropertyItem>::iterator itemIter=m_sourceLabelItems.begin(); + while (itemIter!=m_sourceLabelItems.end()) { + if (itemIter->m_sourceLabelsRHSCount < topNRuleLabelledCount) { + itemIter = m_sourceLabelItems.erase(itemIter); + } else { + std::list< std::pair<size_t,float> >::iterator itemLHSIter=(itemIter->m_sourceLabelsLHSList).begin(); + while (itemLHSIter!=(itemIter->m_sourceLabelsLHSList).end()) { + if (itemLHSIter->second < topNRuleLabelledCount) { + itemLHSIter = (itemIter->m_sourceLabelsLHSList).erase(itemLHSIter); + } else { + if (nKept >= N) { + itemLHSIter = (itemIter->m_sourceLabelsLHSList).erase(itemLHSIter,(itemIter->m_sourceLabelsLHSList).end()); + } else { + ++nKept; + ++itemLHSIter; + } + } + } + if ((itemIter->m_sourceLabelsLHSList).empty()) { + itemIter = m_sourceLabelItems.erase(itemIter); + } else { + ++itemIter; + } + } + } + } +}; + +} // namespace Moses + diff --git a/moses/PP/SourceLabelsPhraseProperty.h b/moses/PP/SourceLabelsPhraseProperty.h new file mode 100644 index 000000000..049d88f17 --- /dev/null +++ b/moses/PP/SourceLabelsPhraseProperty.h @@ -0,0 +1,71 @@ + +#pragma once + +#include "moses/PP/PhraseProperty.h" +#include <string> +#include <list> + +namespace Moses +{ + +// Note that we require label tokens (strings) in the corresponding property values of phrase table entries +// to be replaced beforehand by indices (size_t) of a label vocabulary. (TODO: change that?) + +class SourceLabelsPhrasePropertyItem +{ +friend class SourceLabelsPhraseProperty; + +public: + SourceLabelsPhrasePropertyItem() {}; + + float GetSourceLabelsRHSCount() const + { + return m_sourceLabelsRHSCount; + }; + + const std::list<size_t> &GetSourceLabelsRHS() const + { + return m_sourceLabelsRHS; + }; + + const std::list< std::pair<size_t,float> > &GetSourceLabelsLHSList() const + { + return m_sourceLabelsLHSList; + }; + +private: + float m_sourceLabelsRHSCount; + std::list<size_t> m_sourceLabelsRHS; // should be of size nNTs-1 (empty if initial rule, i.e. no right-hand side non-terminals) + std::list< std::pair<size_t,float> > m_sourceLabelsLHSList; // list of left-hand sides for this right-hand side, with counts +}; + + +class SourceLabelsPhraseProperty : public PhraseProperty +{ +public: + SourceLabelsPhraseProperty(const std::string &value) : PhraseProperty(value) {}; + + virtual void ProcessValue(); + + size_t GetNumberOfNonTerminals() const { + return m_nNTs; + } + + float GetTotalCount() const { + return m_totalCount; + } + + const std::list<SourceLabelsPhrasePropertyItem> &GetSourceLabelItems() const { + return m_sourceLabelItems; + }; + +protected: + + size_t m_nNTs; + float m_totalCount; + + std::list<SourceLabelsPhrasePropertyItem> m_sourceLabelItems; +}; + +} // namespace Moses + |