1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
|
#include <iostream>
#include <string>
#ifdef WITH_THREADS
#include <boost/thread/thread.hpp>
#endif
#include "moses/TranslationModel/CompactPT/LexicalReorderingTableCreator.h"
#include "util/file.hh"
using namespace Moses;
void printHelp(char **argv)
{
std::cerr << "Usage " << argv[0] << ":\n"
" options: \n"
"\t-in string -- input table file name\n"
"\t-out string -- prefix of binary table file\n"
"\t-T string -- path to temporary directory (uses /tmp by default)\n"
#ifdef WITH_THREADS
"\t-threads int|all -- number of threads used for conversion\n"
#endif
"\n advanced:\n"
"\t-landmark int -- use landmark phrase every 2^n phrases\n"
"\t-fingerprint int -- number of bits used for phrase fingerprints\n"
"\t-join-scores -- single set of Huffman codes for score components\n"
"\t-quantize int -- maximum number of scores per score component\n"
"\n"
" For more information see: http://www.statmt.org/moses/?n=Moses.AdvancedFeatures#ntoc6\n\n"
" If you use this please cite:\n\n"
" @article { junczys_pbml98_2012,\n"
" author = { Marcin Junczys-Dowmunt },\n"
" title = { Phrasal Rank-Encoding: Exploiting Phrase Redundancy and\n"
" Translational Relations for Phrase Table Compression },\n"
" journal = { The Prague Bulletin of Mathematical Linguistics },\n"
" volume = { 98 },\n"
" year = { 2012 },\n"
" note = { Proceedings of the MT Marathon 2012, Edinburgh },\n"
" }\n\n"
" Acknowledgments: Part of this research was carried out at and funded by\n"
" the World Intellectual Property Organization (WIPO) in Geneva.\n\n";
}
int main(int argc, char** argv)
{
std::string inFilePath;
std::string outFilePath("out");
std::string tempfilePath;
size_t orderBits = 10;
size_t fingerPrintBits = 16;
bool multipleScoreTrees = true;
size_t quantize = 0;
size_t threads =
#ifdef WITH_THREADS
boost::thread::hardware_concurrency() ? boost::thread::hardware_concurrency() :
#endif
1;
if(1 >= argc) {
printHelp(argv);
return 1;
}
for(int i = 1; i < argc; ++i) {
std::string arg(argv[i]);
if("-in" == arg && i+1 < argc) {
++i;
inFilePath = argv[i];
} else if("-out" == arg && i+1 < argc) {
++i;
outFilePath = argv[i];
} else if("-T" == arg && i+1 < argc) {
++i;
tempfilePath = argv[i];
util::NormalizeTempPrefix(tempfilePath);
} else if("-landmark" == arg && i+1 < argc) {
++i;
orderBits = atoi(argv[i]);
} else if("-fingerprint" == arg && i+1 < argc) {
++i;
fingerPrintBits = atoi(argv[i]);
} else if("-join-scores" == arg) {
multipleScoreTrees = false;
} else if("-quantize" == arg && i+1 < argc) {
++i;
quantize = atoi(argv[i]);
} else if("-threads" == arg && i+1 < argc) {
#ifdef WITH_THREADS
++i;
if(std::string(argv[i]) == "all") {
threads = boost::thread::hardware_concurrency();
if(!threads) {
std::cerr << "Could not determine number of hardware threads, setting to 1" << std::endl;
threads = 1;
}
} else
threads = atoi(argv[i]);
#else
std::cerr << "Thread support not compiled in" << std::endl;
exit(1);
#endif
} else {
//somethings wrong... print help
printHelp(argv);
return 1;
}
}
if(outFilePath.rfind(".minlexr") != outFilePath.size() - 8)
outFilePath += ".minlexr";
LexicalReorderingTableCreator(
inFilePath, outFilePath, tempfilePath,
orderBits, fingerPrintBits,
multipleScoreTrees, quantize
#ifdef WITH_THREADS
, threads
#endif
);
}
|
