misc/pruneGeneration.h


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45

#pragma once
#include <vector>
#include <string>

class Rec
{
public:
  float prob;
  std::string line;
  
  Rec(float aprob, const std::string &aline)
  :prob(aprob)
  ,line(aline)
  {}
  
  inline bool operator< (const Rec &compare) const {
    return prob < compare.prob;
  }
};

////////////////////////////////////////////////////////////

void Output(int limit, std::vector<Rec> &records);
void Prune(int limit, std::vector<Rec> &records);

////////////////////////////////////////////////////////////
inline void Tokenize(std::vector<std::string> &output
                     , const std::string& str
                     , const std::string& delimiters = " \t")
{
  // Skip delimiters at beginning.
  std::string::size_type lastPos = str.find_first_not_of(delimiters, 0);
  // Find first "non-delimiter".
  std::string::size_type pos     = str.find_first_of(delimiters, lastPos);

  while (std::string::npos != pos || std::string::npos != lastPos) {
    // Found a token, add it to the vector.
    output.push_back(str.substr(lastPos, pos - lastPos));
    // Skip delimiters.  Note the "not_of"
    lastPos = str.find_first_not_of(delimiters, pos);
    // Find next "non-delimiter"
    pos = str.find_first_of(delimiters, lastPos);
  }
}