1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
|
#include "Prob.h"
#include "Ngram.h"
#include "Vocab.h"
#include <sstream>
#include <string>
#include <iostream>
#include <stdio.h>
#include <unistd.h>
#include <sys/socket.h>
#include <sys/types.h>
#include <netinet/in.h>
#include <netdb.h>
#include <string.h>
#include <map>
struct Cache {
map<int, Cache> tree;
float prob;
Cache() : prob(0) {}
};
struct LMClient {
Vocab* voc;
int sock, port;
char *s;
struct hostent *hp;
struct sockaddr_in server;
char res[8];
LMClient(Vocab* v, const char* host) : voc(v), port(6666) {
s = strchr(host, ':');
if (s != NULL) {
*s = '\0';
s+=1;
port = atoi(s);
}
sock = socket(AF_INET, SOCK_STREAM, 0);
hp = gethostbyname(host);
if (hp == NULL) {
fprintf(stderr, "unknown host %s\n", host);
exit(1);
}
bzero((char *)&server, sizeof(server));
bcopy(hp->h_addr, (char *)&server.sin_addr, hp->h_length);
server.sin_family = hp->h_addrtype;
server.sin_port = htons(port);
int errors = 0;
while (connect(sock, (struct sockaddr *)&server, sizeof(server)) < 0) {
cerr << "Error: connect()\n";
sleep(1);
errors++;
if (errors > 5) exit(1);
}
std::cerr << "Connected to LM on " << host << " on port " << port << std::endl;
}
float wordProb(int word, int* context) {
Cache* cur = &cache;
int i = 0;
while (context[i] > 0) {
cur = &cur->tree[context[i++]];
}
cur = &cur->tree[word];
if (cur->prob) { return cur->prob; }
i = 0;
ostringstream os;
os << "prob " << voc->getWord((VocabIndex)word);
while (context[i] > 0) {
os << ' ' << voc->getWord((VocabIndex)context[i++]);
}
os << endl;
string out = os.str();
write(sock, out.c_str(), out.size());
int r = read(sock, res, 6);
int errors = 0;
int cnt = 0;
while (1) {
if (r < 0) {
errors++; sleep(1);
cerr << "Error: read()\n";
if (errors > 5) exit(1);
} else if (r==0 || res[cnt] == '\n') { break; }
else {
cnt += r;
if (cnt==6) break;
read(sock, &res[cnt], 6-cnt);
}
}
cur->prob = *reinterpret_cast<float*>(res);
return cur->prob;
}
void clear() {
cache.tree.clear();
}
Cache cache;
};
|
