moses/TranslationModel/ProbingPT/hash.cpp


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44

#include <iostream>
#include "hash.hh"

using namespace std;

namespace Moses
{

uint64_t getHash(StringPiece text)
{
  std::size_t len = text.size();
  uint64_t key = util::MurmurHashNative(text.data(), len);
  return key;
}

std::vector<uint64_t> getVocabIDs(const StringPiece &textin)
{
  //Tokenize
  std::vector<uint64_t> output;

  util::TokenIter<util::SingleCharacter> itWord(textin, util::SingleCharacter(' '));

  while (itWord) {
	StringPiece word = *itWord;
	uint64_t id = 0;

	util::TokenIter<util::SingleCharacter> itFactor(word, util::SingleCharacter('|'));
    while (itFactor) {
    	StringPiece factor = *itFactor;
    	//cerr << "factor=" << factor << endl;

    	id += getHash(factor);
        itFactor++;
    }

    output.push_back(id);
    itWord++;
  }

  return output;
}

}