1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
|
#ifndef ACTIVATION_FUNCTION_H
#define ACTIVATION_FUNCTION_H
#include <cmath>
#include <string>
#include <Eigen/Dense>
#include "util.h"
namespace nplm
{
// is this cheating?
using Eigen::Matrix;
using Eigen::MatrixBase;
enum activation_function_type { Tanh, HardTanh, Rectifier, Identity, InvalidFunction };
inline activation_function_type string_to_activation_function (const std::string &s)
{
if (s == "identity")
return Identity;
else if (s == "rectifier")
return Rectifier;
else if (s == "tanh")
return Tanh;
else if (s == "hardtanh")
return HardTanh;
else
return InvalidFunction;
}
inline std::string activation_function_to_string (activation_function_type f)
{
if (f == Identity)
return "identity";
else if (f == Rectifier)
return "rectifier";
else if (f == Tanh)
return "tanh";
else if (f == HardTanh)
return "hardtanh";
}
struct hardtanh_functor {
user_data_t operator() (user_data_t x) const { if (x < -1.) return -1.; else if (x > 1.) return 1.; else return x; }
};
struct dhardtanh_functor {
user_data_t operator() (user_data_t x) const { return x > -1. && x < 1. ? 1. : 0.; }
};
struct tanh_functor {
user_data_t operator() (user_data_t x) const { return std::tanh(x); }
};
struct dtanh_functor {
user_data_t operator() (user_data_t x) const { return 1-x*x; }
};
struct rectifier_functor {
user_data_t operator() (user_data_t x) const { return std::max(x, user_data_t(0.)); }
};
struct drectifier_functor {
user_data_t operator() (user_data_t x) const { return x > 0. ? 1. : 0.; }
};
class Activation_function
{
int size;
activation_function_type f;
public:
Activation_function() : size(0), f(Rectifier) { }
void resize(int size) { this->size = size; }
void set_activation_function(activation_function_type f) { this->f = f; }
template <typename Engine>
void initialize(Engine &engine, bool init_normal, user_data_t init_range) { }
int n_inputs () const { return size; }
int n_outputs () const { return size; }
template <typename DerivedIn, typename DerivedOut>
void fProp(const MatrixBase<DerivedIn> &input, const MatrixBase<DerivedOut> &output) const
{
UNCONST(DerivedOut, output, my_output);
switch (f)
{
case Identity: my_output = input; break;
case Rectifier: my_output = input.unaryExpr(rectifier_functor()); break;
case Tanh: my_output = input.unaryExpr(tanh_functor()); break;
case HardTanh: my_output = input.unaryExpr(hardtanh_functor()); break;
case InvalidFunction: std::abort();
}
}
template <typename DerivedGOut, typename DerivedGIn, typename DerivedIn, typename DerivedOut>
void bProp(const MatrixBase<DerivedGOut> &input,
MatrixBase<DerivedGIn> &output,
const MatrixBase<DerivedIn> &finput,
const MatrixBase<DerivedOut> &foutput) const
{
UNCONST(DerivedGIn, output, my_output);
switch (f)
{
case Identity: my_output = input; break;
case Rectifier: my_output = finput.array().unaryExpr(drectifier_functor()) * input.array(); break;
case Tanh: my_output = foutput.array().unaryExpr(tanh_functor()) * input.array(); break;
case HardTanh: my_output = finput.array().unaryExpr(hardtanh_functor()) * input.array(); break;
case InvalidFunction: std::abort();
}
}
};
} // namespace nplm
#endif
|
