From a8a5b43f2dc32bd1b45006fd43989dc71e74ba0e Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Tue, 31 Jan 2017 22:21:59 +0000 Subject: move moses2 to root --- moses2/defer/CubePruningPerBitmap/Search.cpp | 273 +++++++++++++++++++++++++++ 1 file changed, 273 insertions(+) create mode 100644 moses2/defer/CubePruningPerBitmap/Search.cpp (limited to 'moses2/defer/CubePruningPerBitmap/Search.cpp') diff --git a/moses2/defer/CubePruningPerBitmap/Search.cpp b/moses2/defer/CubePruningPerBitmap/Search.cpp new file mode 100644 index 000000000..b0eddcc21 --- /dev/null +++ b/moses2/defer/CubePruningPerBitmap/Search.cpp @@ -0,0 +1,273 @@ +/* + * Search.cpp + * + * Created on: 16 Nov 2015 + * Author: hieu + */ +#include +#include "Search.h" +#include "../Manager.h" +#include "../Hypothesis.h" +#include "../../InputPaths.h" +#include "../../InputPath.h" +#include "../../System.h" +#include "../../Sentence.h" +#include "../../TranslationTask.h" +#include "../../legacy/Util2.h" + +using namespace std; + +namespace Moses2 +{ + +namespace NSCubePruningPerBitmap +{ + +//////////////////////////////////////////////////////////////////////// +Search::Search(Manager &mgr) +:Moses2::Search(mgr) +,m_stacks(mgr) + +,m_queue(QueueItemOrderer(), + std::vector() ) + +,m_seenPositions() +{ +} + +Search::~Search() +{ +} + +void Search::Decode() +{ + // init stacks + m_stacks.Init(mgr.GetInput().GetSize() + 1); + + const Bitmap &initBitmap = mgr.GetBitmaps().GetInitialBitmap(); + Hypothesis *initHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr); + initHypo->Init(mgr, mgr.GetInputPaths().GetBlank(), mgr.GetInitPhrase(), initBitmap); + initHypo->EmptyHypothesisState(mgr.GetInput()); + + m_stacks.Add(initHypo, mgr.GetHypoRecycle()); + + for (size_t stackInd = 0; stackInd < m_stacks.GetSize() - 1; ++stackInd) { + CreateSearchGraph(stackInd); + } + + for (size_t stackInd = 1; stackInd < m_stacks.GetSize(); ++stackInd) { + //cerr << "stackInd=" << stackInd << endl; + Decode(stackInd); + + //cerr << m_stacks << endl; + } + + //DebugCounts(); +} + +void Search::Decode(size_t stackInd) +{ + NSCubePruningMiniStack::Stack &stack = m_stacks[stackInd]; + + // FOR EACH BITMAP IN EACH STACK + boost::unordered_map > uniqueBM; + + BOOST_FOREACH(NSCubePruningMiniStack::Stack::Coll::value_type &val, stack.GetColl()) { + NSCubePruningMiniStack::MiniStack &miniStack = *val.second; + + const Bitmap *bitmap = val.first.first; + uniqueBM[bitmap].push_back(&miniStack); + } + + // decode each bitmap + boost::unordered_map >::iterator iter; + for (iter = uniqueBM.begin(); iter != uniqueBM.end(); ++iter) { + const vector &miniStacks = iter->second; + Decode(miniStacks); + } + + /* + // FOR EACH STACK + vector miniStacks; + BOOST_FOREACH(NSCubePruningMiniStack::Stack::Coll::value_type &val, stack.GetColl()) { + NSCubePruningMiniStack::MiniStack &miniStack = *val.second; + + miniStacks.push_back(&miniStack); + } + Decode(miniStacks); + */ +} + +void Search::Decode(const vector &miniStacks) +{ + Recycler &hypoRecycler = mgr.GetHypoRecycle(); + + // reuse queue from previous stack. Clear it first + std::vector &container = Container(m_queue); + //cerr << "container=" << container.size() << endl; + BOOST_FOREACH(QueueItem *item, container) { + // recycle unused hypos from queue + Hypothesis *hypo = item->hypo; + hypoRecycler.Recycle(hypo); + + // recycle queue item + m_queueItemRecycler.push_back(item); + } + container.clear(); + + m_seenPositions.clear(); + + BOOST_FOREACH(NSCubePruningMiniStack::MiniStack *miniStack, miniStacks) { + // add top hypo from every edge into queue + CubeEdges &edges = *m_cubeEdges[miniStack]; + + BOOST_FOREACH(CubeEdge *edge, edges) { + //cerr << "edge=" << *edge << endl; + edge->CreateFirst(mgr, m_queue, m_seenPositions, m_queueItemRecycler); + } + } + + size_t pops = 0; + while (!m_queue.empty() && pops < mgr.system.popLimit) { + // get best hypo from queue, add to stack + //cerr << "queue=" << queue.size() << endl; + QueueItem *item = m_queue.top(); + m_queue.pop(); + + CubeEdge *edge = item->edge; + + // add hypo to stack + Hypothesis *hypo = item->hypo; + //cerr << "hypo=" << *hypo << " " << hypo->GetBitmap() << endl; + m_stacks.Add(hypo, hypoRecycler); + + edge->CreateNext(mgr, item, m_queue, m_seenPositions, m_queueItemRecycler); + + ++pops; + } + + /* + // create hypo from every edge. Increase diversity + while (!m_queue.empty()) { + QueueItem *item = m_queue.top(); + m_queue.pop(); + + if (item->hypoIndex == 0 && item->tpIndex == 0) { + CubeEdge &edge = item->edge; + + // add hypo to stack + Hypothesis *hypo = item->hypo; + //cerr << "hypo=" << *hypo << " " << hypo->GetBitmap() << endl; + m_stacks.Add(hypo, mgr.GetHypoRecycle()); + } + } + */ +} + + +void Search::CreateSearchGraph(size_t stackInd) +{ + NSCubePruningMiniStack::Stack &stack = m_stacks[stackInd]; + MemPool &pool = mgr.GetPool(); + + BOOST_FOREACH(const NSCubePruningMiniStack::Stack::Coll::value_type &val, stack.GetColl()) { + const Bitmap &hypoBitmap = *val.first.first; + size_t hypoEndPos = val.first.second; + //cerr << "key=" << hypoBitmap << " " << hypoEndPos << endl; + + // create edges to next hypos from existing hypos + const InputPaths &paths = mgr.GetInputPaths(); + + BOOST_FOREACH(const InputPath *path, paths) { + const Range &pathRange = path->range; + //cerr << "pathRange=" << pathRange << endl; + + if (!path->IsUsed()) { + continue; + } + if (!CanExtend(hypoBitmap, hypoEndPos, pathRange)) { + continue; + } + + const Bitmap &newBitmap = mgr.GetBitmaps().GetBitmap(hypoBitmap, pathRange); + + // sort hypo for a particular bitmap and hypoEndPos + const NSCubePruningMiniStack::MiniStack &miniStack = *val.second; + + + // add cube edge + size_t numPt = mgr.system.mappings.size(); + for (size_t i = 0; i < numPt; ++i) { + const TargetPhrases *tps = path->targetPhrases[i]; + if (tps && tps->GetSize()) { + // create next mini stack + NSCubePruningMiniStack::MiniStack &nextMiniStack = m_stacks.GetMiniStack(newBitmap, pathRange); + + CubeEdge *edge = new (pool.Allocate()) CubeEdge(mgr, miniStack, *path, *tps, newBitmap); + + CubeEdges *edges; + boost::unordered_map::iterator iter = m_cubeEdges.find(&nextMiniStack); + if (iter == m_cubeEdges.end()) { + edges = new (pool.Allocate()) CubeEdges(); + m_cubeEdges[&nextMiniStack] = edges; + } + else { + edges = iter->second; + } + + edges->push_back(edge); + } + } + } + } + +} + + +const Hypothesis *Search::GetBestHypo() const +{ + const NSCubePruningMiniStack::Stack &lastStack = m_stacks.Back(); + std::vector sortedHypos = lastStack.GetBestHypos(1); + + const Hypothesis *best = NULL; + if (sortedHypos.size()) { + best = sortedHypos[0]; + } + return best; +} + +void Search::DebugCounts() +{ + std::map counts; + + for (size_t stackInd = 0; stackInd < m_stacks.GetSize(); ++stackInd) { + //cerr << "stackInd=" << stackInd << endl; + const NSCubePruningMiniStack::Stack &stack = m_stacks[stackInd]; + BOOST_FOREACH(const NSCubePruningMiniStack::Stack::Coll::value_type &val, stack.GetColl()) { + const NSCubePruningMiniStack::MiniStack &miniStack = *val.second; + size_t count = miniStack.GetColl().size(); + + if (counts.find(count) == counts.end()) { + counts[count] = 0; + } + else { + ++counts[count]; + } + } + //cerr << m_stacks << endl; + } + + std::map::const_iterator iter; + for (iter = counts.begin(); iter != counts.end(); ++iter) { + cerr << iter->first << "=" << iter->second << " "; + } + cerr << endl; +} + + + +} + +} + + -- cgit v1.2.3 From 84b918b389e5a2d5e31cca993e1e53ff1354f1b1 Mon Sep 17 00:00:00 2001 From: Hieu Hoang Date: Wed, 1 Feb 2017 00:27:14 +0000 Subject: beautify --- moses2/defer/CubePruningPerBitmap/Search.cpp | 338 +++++++++++++-------------- 1 file changed, 168 insertions(+), 170 deletions(-) (limited to 'moses2/defer/CubePruningPerBitmap/Search.cpp') diff --git a/moses2/defer/CubePruningPerBitmap/Search.cpp b/moses2/defer/CubePruningPerBitmap/Search.cpp index b0eddcc21..d07b28a72 100644 --- a/moses2/defer/CubePruningPerBitmap/Search.cpp +++ b/moses2/defer/CubePruningPerBitmap/Search.cpp @@ -25,13 +25,13 @@ namespace NSCubePruningPerBitmap //////////////////////////////////////////////////////////////////////// Search::Search(Manager &mgr) -:Moses2::Search(mgr) -,m_stacks(mgr) + :Moses2::Search(mgr) + ,m_stacks(mgr) -,m_queue(QueueItemOrderer(), - std::vector() ) + ,m_queue(QueueItemOrderer(), + std::vector() ) -,m_seenPositions() + ,m_seenPositions() { } @@ -41,28 +41,28 @@ Search::~Search() void Search::Decode() { - // init stacks - m_stacks.Init(mgr.GetInput().GetSize() + 1); + // init stacks + m_stacks.Init(mgr.GetInput().GetSize() + 1); - const Bitmap &initBitmap = mgr.GetBitmaps().GetInitialBitmap(); - Hypothesis *initHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr); - initHypo->Init(mgr, mgr.GetInputPaths().GetBlank(), mgr.GetInitPhrase(), initBitmap); - initHypo->EmptyHypothesisState(mgr.GetInput()); + const Bitmap &initBitmap = mgr.GetBitmaps().GetInitialBitmap(); + Hypothesis *initHypo = Hypothesis::Create(mgr.GetSystemPool(), mgr); + initHypo->Init(mgr, mgr.GetInputPaths().GetBlank(), mgr.GetInitPhrase(), initBitmap); + initHypo->EmptyHypothesisState(mgr.GetInput()); - m_stacks.Add(initHypo, mgr.GetHypoRecycle()); + m_stacks.Add(initHypo, mgr.GetHypoRecycle()); - for (size_t stackInd = 0; stackInd < m_stacks.GetSize() - 1; ++stackInd) { - CreateSearchGraph(stackInd); - } + for (size_t stackInd = 0; stackInd < m_stacks.GetSize() - 1; ++stackInd) { + CreateSearchGraph(stackInd); + } - for (size_t stackInd = 1; stackInd < m_stacks.GetSize(); ++stackInd) { - //cerr << "stackInd=" << stackInd << endl; - Decode(stackInd); + for (size_t stackInd = 1; stackInd < m_stacks.GetSize(); ++stackInd) { + //cerr << "stackInd=" << stackInd << endl; + Decode(stackInd); - //cerr << m_stacks << endl; - } + //cerr << m_stacks << endl; + } - //DebugCounts(); + //DebugCounts(); } void Search::Decode(size_t stackInd) @@ -73,26 +73,26 @@ void Search::Decode(size_t stackInd) boost::unordered_map > uniqueBM; BOOST_FOREACH(NSCubePruningMiniStack::Stack::Coll::value_type &val, stack.GetColl()) { - NSCubePruningMiniStack::MiniStack &miniStack = *val.second; + NSCubePruningMiniStack::MiniStack &miniStack = *val.second; - const Bitmap *bitmap = val.first.first; - uniqueBM[bitmap].push_back(&miniStack); + const Bitmap *bitmap = val.first.first; + uniqueBM[bitmap].push_back(&miniStack); } // decode each bitmap boost::unordered_map >::iterator iter; for (iter = uniqueBM.begin(); iter != uniqueBM.end(); ++iter) { - const vector &miniStacks = iter->second; - Decode(miniStacks); + const vector &miniStacks = iter->second; + Decode(miniStacks); } /* // FOR EACH STACK vector miniStacks; BOOST_FOREACH(NSCubePruningMiniStack::Stack::Coll::value_type &val, stack.GetColl()) { - NSCubePruningMiniStack::MiniStack &miniStack = *val.second; + NSCubePruningMiniStack::MiniStack &miniStack = *val.second; - miniStacks.push_back(&miniStack); + miniStacks.push_back(&miniStack); } Decode(miniStacks); */ @@ -100,68 +100,68 @@ void Search::Decode(size_t stackInd) void Search::Decode(const vector &miniStacks) { - Recycler &hypoRecycler = mgr.GetHypoRecycle(); - - // reuse queue from previous stack. Clear it first - std::vector &container = Container(m_queue); - //cerr << "container=" << container.size() << endl; - BOOST_FOREACH(QueueItem *item, container) { - // recycle unused hypos from queue - Hypothesis *hypo = item->hypo; - hypoRecycler.Recycle(hypo); - - // recycle queue item - m_queueItemRecycler.push_back(item); - } - container.clear(); - - m_seenPositions.clear(); - - BOOST_FOREACH(NSCubePruningMiniStack::MiniStack *miniStack, miniStacks) { - // add top hypo from every edge into queue - CubeEdges &edges = *m_cubeEdges[miniStack]; - - BOOST_FOREACH(CubeEdge *edge, edges) { - //cerr << "edge=" << *edge << endl; - edge->CreateFirst(mgr, m_queue, m_seenPositions, m_queueItemRecycler); - } - } - - size_t pops = 0; - while (!m_queue.empty() && pops < mgr.system.popLimit) { - // get best hypo from queue, add to stack - //cerr << "queue=" << queue.size() << endl; - QueueItem *item = m_queue.top(); - m_queue.pop(); - - CubeEdge *edge = item->edge; - - // add hypo to stack - Hypothesis *hypo = item->hypo; - //cerr << "hypo=" << *hypo << " " << hypo->GetBitmap() << endl; - m_stacks.Add(hypo, hypoRecycler); - - edge->CreateNext(mgr, item, m_queue, m_seenPositions, m_queueItemRecycler); - - ++pops; - } - - /* - // create hypo from every edge. Increase diversity - while (!m_queue.empty()) { - QueueItem *item = m_queue.top(); - m_queue.pop(); - - if (item->hypoIndex == 0 && item->tpIndex == 0) { - CubeEdge &edge = item->edge; - - // add hypo to stack - Hypothesis *hypo = item->hypo; - //cerr << "hypo=" << *hypo << " " << hypo->GetBitmap() << endl; - m_stacks.Add(hypo, mgr.GetHypoRecycle()); - } - } - */ + Recycler &hypoRecycler = mgr.GetHypoRecycle(); + + // reuse queue from previous stack. Clear it first + std::vector &container = Container(m_queue); + //cerr << "container=" << container.size() << endl; + BOOST_FOREACH(QueueItem *item, container) { + // recycle unused hypos from queue + Hypothesis *hypo = item->hypo; + hypoRecycler.Recycle(hypo); + + // recycle queue item + m_queueItemRecycler.push_back(item); + } + container.clear(); + + m_seenPositions.clear(); + + BOOST_FOREACH(NSCubePruningMiniStack::MiniStack *miniStack, miniStacks) { + // add top hypo from every edge into queue + CubeEdges &edges = *m_cubeEdges[miniStack]; + + BOOST_FOREACH(CubeEdge *edge, edges) { + //cerr << "edge=" << *edge << endl; + edge->CreateFirst(mgr, m_queue, m_seenPositions, m_queueItemRecycler); + } + } + + size_t pops = 0; + while (!m_queue.empty() && pops < mgr.system.popLimit) { + // get best hypo from queue, add to stack + //cerr << "queue=" << queue.size() << endl; + QueueItem *item = m_queue.top(); + m_queue.pop(); + + CubeEdge *edge = item->edge; + + // add hypo to stack + Hypothesis *hypo = item->hypo; + //cerr << "hypo=" << *hypo << " " << hypo->GetBitmap() << endl; + m_stacks.Add(hypo, hypoRecycler); + + edge->CreateNext(mgr, item, m_queue, m_seenPositions, m_queueItemRecycler); + + ++pops; + } + + /* + // create hypo from every edge. Increase diversity + while (!m_queue.empty()) { + QueueItem *item = m_queue.top(); + m_queue.pop(); + + if (item->hypoIndex == 0 && item->tpIndex == 0) { + CubeEdge &edge = item->edge; + + // add hypo to stack + Hypothesis *hypo = item->hypo; + //cerr << "hypo=" << *hypo << " " << hypo->GetBitmap() << endl; + m_stacks.Add(hypo, mgr.GetHypoRecycle()); + } + } + */ } @@ -171,54 +171,53 @@ void Search::CreateSearchGraph(size_t stackInd) MemPool &pool = mgr.GetPool(); BOOST_FOREACH(const NSCubePruningMiniStack::Stack::Coll::value_type &val, stack.GetColl()) { - const Bitmap &hypoBitmap = *val.first.first; - size_t hypoEndPos = val.first.second; - //cerr << "key=" << hypoBitmap << " " << hypoEndPos << endl; - - // create edges to next hypos from existing hypos - const InputPaths &paths = mgr.GetInputPaths(); - - BOOST_FOREACH(const InputPath *path, paths) { - const Range &pathRange = path->range; - //cerr << "pathRange=" << pathRange << endl; - - if (!path->IsUsed()) { - continue; - } - if (!CanExtend(hypoBitmap, hypoEndPos, pathRange)) { - continue; - } - - const Bitmap &newBitmap = mgr.GetBitmaps().GetBitmap(hypoBitmap, pathRange); - - // sort hypo for a particular bitmap and hypoEndPos - const NSCubePruningMiniStack::MiniStack &miniStack = *val.second; - - - // add cube edge - size_t numPt = mgr.system.mappings.size(); - for (size_t i = 0; i < numPt; ++i) { - const TargetPhrases *tps = path->targetPhrases[i]; - if (tps && tps->GetSize()) { - // create next mini stack - NSCubePruningMiniStack::MiniStack &nextMiniStack = m_stacks.GetMiniStack(newBitmap, pathRange); - - CubeEdge *edge = new (pool.Allocate()) CubeEdge(mgr, miniStack, *path, *tps, newBitmap); - - CubeEdges *edges; - boost::unordered_map::iterator iter = m_cubeEdges.find(&nextMiniStack); - if (iter == m_cubeEdges.end()) { - edges = new (pool.Allocate()) CubeEdges(); - m_cubeEdges[&nextMiniStack] = edges; - } - else { - edges = iter->second; - } - - edges->push_back(edge); - } - } - } + const Bitmap &hypoBitmap = *val.first.first; + size_t hypoEndPos = val.first.second; + //cerr << "key=" << hypoBitmap << " " << hypoEndPos << endl; + + // create edges to next hypos from existing hypos + const InputPaths &paths = mgr.GetInputPaths(); + + BOOST_FOREACH(const InputPath *path, paths) { + const Range &pathRange = path->range; + //cerr << "pathRange=" << pathRange << endl; + + if (!path->IsUsed()) { + continue; + } + if (!CanExtend(hypoBitmap, hypoEndPos, pathRange)) { + continue; + } + + const Bitmap &newBitmap = mgr.GetBitmaps().GetBitmap(hypoBitmap, pathRange); + + // sort hypo for a particular bitmap and hypoEndPos + const NSCubePruningMiniStack::MiniStack &miniStack = *val.second; + + + // add cube edge + size_t numPt = mgr.system.mappings.size(); + for (size_t i = 0; i < numPt; ++i) { + const TargetPhrases *tps = path->targetPhrases[i]; + if (tps && tps->GetSize()) { + // create next mini stack + NSCubePruningMiniStack::MiniStack &nextMiniStack = m_stacks.GetMiniStack(newBitmap, pathRange); + + CubeEdge *edge = new (pool.Allocate()) CubeEdge(mgr, miniStack, *path, *tps, newBitmap); + + CubeEdges *edges; + boost::unordered_map::iterator iter = m_cubeEdges.find(&nextMiniStack); + if (iter == m_cubeEdges.end()) { + edges = new (pool.Allocate()) CubeEdges(); + m_cubeEdges[&nextMiniStack] = edges; + } else { + edges = iter->second; + } + + edges->push_back(edge); + } + } + } } } @@ -226,42 +225,41 @@ void Search::CreateSearchGraph(size_t stackInd) const Hypothesis *Search::GetBestHypo() const { - const NSCubePruningMiniStack::Stack &lastStack = m_stacks.Back(); - std::vector sortedHypos = lastStack.GetBestHypos(1); - - const Hypothesis *best = NULL; - if (sortedHypos.size()) { - best = sortedHypos[0]; - } - return best; + const NSCubePruningMiniStack::Stack &lastStack = m_stacks.Back(); + std::vector sortedHypos = lastStack.GetBestHypos(1); + + const Hypothesis *best = NULL; + if (sortedHypos.size()) { + best = sortedHypos[0]; + } + return best; } void Search::DebugCounts() { - std::map counts; - - for (size_t stackInd = 0; stackInd < m_stacks.GetSize(); ++stackInd) { - //cerr << "stackInd=" << stackInd << endl; - const NSCubePruningMiniStack::Stack &stack = m_stacks[stackInd]; - BOOST_FOREACH(const NSCubePruningMiniStack::Stack::Coll::value_type &val, stack.GetColl()) { - const NSCubePruningMiniStack::MiniStack &miniStack = *val.second; - size_t count = miniStack.GetColl().size(); - - if (counts.find(count) == counts.end()) { - counts[count] = 0; - } - else { - ++counts[count]; - } - } - //cerr << m_stacks << endl; - } - - std::map::const_iterator iter; - for (iter = counts.begin(); iter != counts.end(); ++iter) { - cerr << iter->first << "=" << iter->second << " "; - } - cerr << endl; + std::map counts; + + for (size_t stackInd = 0; stackInd < m_stacks.GetSize(); ++stackInd) { + //cerr << "stackInd=" << stackInd << endl; + const NSCubePruningMiniStack::Stack &stack = m_stacks[stackInd]; + BOOST_FOREACH(const NSCubePruningMiniStack::Stack::Coll::value_type &val, stack.GetColl()) { + const NSCubePruningMiniStack::MiniStack &miniStack = *val.second; + size_t count = miniStack.GetColl().size(); + + if (counts.find(count) == counts.end()) { + counts[count] = 0; + } else { + ++counts[count]; + } + } + //cerr << m_stacks << endl; + } + + std::map::const_iterator iter; + for (iter = counts.begin(); iter != counts.end(); ++iter) { + cerr << iter->first << "=" << iter->second << " "; + } + cerr << endl; } -- cgit v1.2.3