1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
|
#ifndef MODEL_H
#define MODEL_H
#include <iostream>
#include <vector>
#include <string>
#include <boost/random/mersenne_twister.hpp>
#include "neuralClasses.h"
#include "Activation_function.h"
namespace nplm
{
class model {
public:
Input_word_embeddings input_layer;
Linear_layer first_hidden_linear;
Activation_function first_hidden_activation;
Linear_layer second_hidden_linear;
Activation_function second_hidden_activation;
Output_word_embeddings output_layer;
Matrix<double,Dynamic,Dynamic,Eigen::RowMajor> output_embedding_matrix,
input_embedding_matrix,
input_and_output_embedding_matrix;
activation_function_type activation_function;
int ngram_size, input_vocab_size, output_vocab_size, input_embedding_dimension, num_hidden, output_embedding_dimension;
bool premultiplied;
model(int ngram_size,
int input_vocab_size,
int output_vocab_size,
int input_embedding_dimension,
int num_hidden,
int output_embedding_dimension,
bool share_embeddings)
{
if (share_embeddings){
input_and_output_embedding_matrix = Matrix<double,Dynamic,Dynamic,Eigen::RowMajor>();
input_layer.set_W(&input_and_output_embedding_matrix);
output_layer.set_W(&input_and_output_embedding_matrix);
}
else {
input_embedding_matrix = Matrix<double,Dynamic,Dynamic,Eigen::RowMajor>();
output_embedding_matrix = Matrix<double,Dynamic,Dynamic,Eigen::RowMajor>();
input_layer.set_W(&input_embedding_matrix);
output_layer.set_W(&output_embedding_matrix);
}
resize(ngram_size,
input_vocab_size,
output_vocab_size,
input_embedding_dimension,
num_hidden,
output_embedding_dimension);
}
model() : ngram_size(1),
premultiplied(false),
activation_function(Rectifier),
output_embedding_matrix(Matrix<double,Dynamic,Dynamic,Eigen::RowMajor>()),
input_embedding_matrix(Matrix<double,Dynamic,Dynamic,Eigen::RowMajor>())
{
output_layer.set_W(&output_embedding_matrix);
input_layer.set_W(&input_embedding_matrix);
}
void resize(int ngram_size,
int input_vocab_size,
int output_vocab_size,
int input_embedding_dimension,
int num_hidden,
int output_embedding_dimension);
void initialize(boost::random::mt19937 &init_engine,
bool init_normal,
double init_range,
double init_bias,
string ¶meter_udpate,
double adagrad_epsilon);
void set_activation_function(activation_function_type f)
{
activation_function = f;
first_hidden_activation.set_activation_function(f);
second_hidden_activation.set_activation_function(f);
}
void premultiply();
// Since the vocabulary is not essential to the model,
// we need a version with and without a vocabulary.
// If the number of "extra" data structures like this grows,
// a better solution is needed
void read(const std::string &filename);
void read(const std::string &filename, std::vector<std::string> &words);
void read(const std::string &filename, std::vector<std::string> &input_words, std::vector<std::string> &output_words);
void write(const std::string &filename);
void write(const std::string &filename, const std::vector<std::string> &words);
void write(const std::string &filename, const std::vector<std::string> &input_words, const std::vector<std::string> &output_words);
private:
void readConfig(std::ifstream &config_file);
void readConfig(const std::string &filename);
void write(const std::string &filename, const std::vector<std::string> *input_pwords, const std::vector<std::string> *output_pwords);
};
} //namespace nplm
#endif
|
