blob: 85574a40a2f5fb1b2cba27e0a50640969811cb23 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
|
#pragma once
#include "search/idf_map.hpp"
#include "base/assert.hpp"
#include "base/string_utils.hpp"
#include <algorithm>
#include <cmath>
#include <cstddef>
#include <cstdint>
#include <optional>
#include <sstream>
#include <string>
#include <utility>
namespace search
{
class IdfMap;
struct TokenFrequencyPair
{
TokenFrequencyPair() = default;
template <typename Token>
TokenFrequencyPair(Token && token, uint64_t frequency)
: m_token(std::forward<Token>(token)), m_frequency(frequency)
{
}
bool operator<(TokenFrequencyPair const & rhs) const;
void Swap(TokenFrequencyPair & rhs);
strings::UniString m_token;
uint64_t m_frequency = 0;
};
std::string DebugPrint(TokenFrequencyPair const & tf);
// This class represents a document in a vector space of tokens.
class DocVec
{
public:
class Builder
{
public:
template <typename Token>
void Add(Token && token)
{
m_tokens.emplace_back(std::forward<Token>(token));
}
private:
friend class DocVec;
std::vector<strings::UniString> m_tokens;
};
DocVec() = default;
explicit DocVec(Builder const & builder);
// Computes vector norm of the doc.
double Norm(IdfMap & idfs) const;
size_t GetNumTokens() const { return m_tfs.size(); }
strings::UniString const & GetToken(size_t i) const;
double GetIdf(IdfMap & idfs, size_t i) const;
double GetWeight(IdfMap & idfs, size_t i) const;
bool Empty() const { return m_tfs.empty(); }
private:
friend std::string DebugPrint(DocVec const & dv)
{
return "DocVec " + ::DebugPrint(dv.m_tfs);
}
std::vector<TokenFrequencyPair> m_tfs;
};
// This class represents a search query in a vector space of tokens.
class QueryVec
{
public:
class Builder
{
public:
template <typename Token>
void AddFull(Token && token)
{
m_tokens.emplace_back(std::forward<Token>(token));
}
template <typename Token>
void SetPrefix(Token && token)
{
m_prefix = std::forward<Token>(token);
}
private:
friend class QueryVec;
std::vector<strings::UniString> m_tokens;
std::optional<strings::UniString> m_prefix;
};
explicit QueryVec(IdfMap & idfs) : m_idfs(&idfs) {}
QueryVec(IdfMap & idfs, Builder const & builder);
// Computes cosine similarity between |*this| and |rhs|.
double Similarity(IdfMap & docIdfs, DocVec const & rhs);
// Computes vector norm of the query.
double Norm();
bool Empty() const { return m_tfs.empty() && !m_prefix; }
private:
double GetFullTokenWeight(size_t i);
double GetPrefixTokenWeight();
friend std::string DebugPrint(QueryVec const & qv)
{
std::ostringstream os;
os << "QueryVec " + ::DebugPrint(qv.m_tfs);
if (qv.m_prefix)
os << " " << DebugPrint(*qv.m_prefix);
return os.str();
}
IdfMap * m_idfs;
std::vector<TokenFrequencyPair> m_tfs;
std::optional<strings::UniString> m_prefix;
};
} // namespace search
|
