1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
|
/*
* SentenceWithCandidates.cpp
*
* Created on: 14 Dec 2015
* Author: hieu
*/
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/xml_parser.hpp>
#include <boost/algorithm/string.hpp>
#include "SentenceWithCandidates.h"
#include "../System.h"
#include "../parameters/AllOptions.h"
#include "../legacy/Util2.h"
#include <unordered_map>
using namespace std;
using namespace boost;
namespace Moses2
{
const string SentenceWithCandidates::INPUT_PART_DELIM = "@@@";
const string SentenceWithCandidates::PT_LINE_DELIM = "$$$";
SentenceWithCandidates *SentenceWithCandidates::CreateFromString(MemPool &pool, FactorCollection &vocab,
const System &system, const std::string &str)
{
SentenceWithCandidates *ret;
// Break input into two parts: the parts are delimited by
typedef split_iterator<string::const_iterator> string_split_iterator;
vector<string> input_parts;
for(string_split_iterator It= make_split_iterator(str, first_finder(SentenceWithCandidates::INPUT_PART_DELIM, is_iequal()));
It!=string_split_iterator();
++It)
{
input_parts.push_back(copy_range<std::string>(*It));
}
//cerr << "Number of subparts: " << input_parts.size() << endl;
if (input_parts.size() ==2 ) {
//cerr << "correct number of parts" << endl ;
} else {
// TODO: how to handle wrong input format
cerr << "INCORRECT number of parts" << endl ;
exit(1);
}
trim(input_parts[0]);
trim(input_parts[1]);
//cerr << "Input String: " << input_parts[0] << endl ;
//cerr << "Phrase Table: " << input_parts[1] << endl ;
///// Process the text part of the input
const string partstr = input_parts[0];
// no xml
//cerr << "PB SentenceWithCandidates" << endl;
std::vector<std::string> toks = Tokenize(partstr);
size_t size = toks.size();
ret = new (pool.Allocate<SentenceWithCandidates>()) SentenceWithCandidates(pool, size);
ret->PhraseImplTemplate<Word>::CreateFromString(vocab, system, toks, false);
//cerr << "REORDERING CONSTRAINTS:" << ret->GetReorderingConstraint() << endl;
//cerr << "ret=" << ret->Debug(system) << endl;
//// Parse the phrase table of the input
ret->m_phraseTableString = replace_all_copy(input_parts[1],PT_LINE_DELIM,"\n");
// ret->m_phraseTableString="constant phrase table";
// cerr << "Extracted Phrase Table String: " << ret->m_phraseTableString << endl;
//cerr << "Extracted Phrase Table String: " << ret->getPhraseTableString() << endl;
return ret;
}
SentenceWithCandidates::SentenceWithCandidates(MemPool &pool, size_t size)
:Sentence(pool, size)
{
//cerr << "SentenceWithCandidates::SentenceWithCandidates" << endl;
}
SentenceWithCandidates::~SentenceWithCandidates()
{
//cerr << "SentenceWithCandidates::~SentenceWithCandidates" << endl;
}
std::string SentenceWithCandidates::Debug(const System &system) const
{
return "SentenceWithCandidates::Debug";
}
} /* namespace Moses2 */
|
