From 3d226d9bd526694ffecb39c6ba15cd2748ccf7ab Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Sun, 29 Oct 2017 12:39:23 +0000 Subject: Add a comment for SwishNodeOp --- src/graph/node_operators_unary.h | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) (limited to 'src/graph/node_operators_unary.h') diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 2c205f17..38241258 100644 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -196,21 +196,19 @@ struct TanhNodeOp : public NaryNodeOp { /** * Represents a rectified -linear node - * in an expression graph. + * href="https://en.wikipedia.org/wiki/Rectifier_(neural_networks)">rectified + * linear node in an expression graph. * - * This node implements the activation function - * \f$f(x) = \max(0, x)\f$ and its derivative: + * This node implements the activationfunction \f$ f(x) = \max(0, x) \f$ and + * its derivative: * - \f[ - f^\prime(x) = - \begin{cases} - 0 & \text{if } x \leq 0 \\ - 1 & \text{if } x > 0 - \end{cases} -\f] + * \f[ + * f^\prime(x) = + * \begin{cases} + * 0 & \text{if } x \leq 0 \\ + * 1 & \text{if } x > 0 + * \end{cases} + * \f] */ struct ReLUNodeOp : public UnaryNodeOp { template @@ -228,6 +226,16 @@ struct ReLUNodeOp : public UnaryNodeOp { const std::string type() { return "ReLU"; } }; +/** + * Represents a swish node + * in an expression graph. + * + * This node implements the activation function + * \f$ f(x) = x \cdot \sigma(x) \f$ + * and its derivative + * \f$ f^\prime(x) = f(x) + \sigma(x)(1 - f(x)) \f$ . + * + */ struct SwishNodeOp : public UnaryNodeOp { template SwishNodeOp(Args... args) : UnaryNodeOp(args...) {} -- cgit v1.2.3 From 505353d76dc95fa3df703b4ff908fa489eb0e85b Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Sun, 29 Oct 2017 14:06:53 +0000 Subject: Add comments to ReLU i Swish --- src/graph/node_operators_unary.h | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) (limited to 'src/graph/node_operators_unary.h') diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 38241258..6a78da67 100644 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -199,9 +199,8 @@ struct TanhNodeOp : public NaryNodeOp { * href="https://en.wikipedia.org/wiki/Rectifier_(neural_networks)">rectified * linear node in an expression graph. * - * This node implements the activationfunction \f$ f(x) = \max(0, x) \f$ and + * This node implements the activation function \f$ f(x) = \max(0, x) \f$ and * its derivative: - * * \f[ * f^\prime(x) = * \begin{cases} @@ -215,12 +214,20 @@ struct ReLUNodeOp : public UnaryNodeOp { ReLUNodeOp(Args... args) : UnaryNodeOp(args...) {} NodeOps forwardOps() { - return {NodeOp(Element(_1 = ReLU(_2), val_, child(0)->val()))}; + // f(x) = max(0, x) + return {NodeOp(Element(_1 = ReLU(_2), + val_, // _1 := f(x) to be calculated + child(0)->val() // _2 := x + ))}; } NodeOps backwardOps() { - return {NodeOp( - Add(_1 * ReLUback(_2), child(0)->grad(), adj_, child(0)->val()))}; + // dJ/dx += dJ/df * binarystep(x) + return {NodeOp(Add(_1 * ReLUback(_2), + child(0)->grad(), // dJ/dx + adj_, // _1 := dJ/df + child(0)->val() // _2 := f(x) = max(0, x) + ))}; } const std::string type() { return "ReLU"; } @@ -245,11 +252,13 @@ struct SwishNodeOp : public UnaryNodeOp { } NodeOps backwardOps() { + // dJ/dx += dJ/df * ( f(x) + sigma(x) * (1 - f(x)) ) return {NodeOp(Add(_1 * (_3 + Sigma(_2) * (1.f - _3)), - child(0)->grad(), - adj_, - child(0)->val(), - val_))}; + child(0)->grad(), // dJ/dx + adj_, // _1 := dJ/df + child(0)->val(), // _2 := x + val_ // _3 := f(x) = x*sigma(x) + ))}; } const std::string type() { return "swish"; } -- cgit v1.2.3 From 2038b3cb2dbfa81876c71de6eb4bd953b8ef7587 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Sun, 29 Oct 2017 14:43:28 +0000 Subject: Add LeakyReLU --- src/graph/node_operators_unary.h | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) (limited to 'src/graph/node_operators_unary.h') diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 6a78da67..93c12a3c 100644 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -233,6 +233,46 @@ struct ReLUNodeOp : public UnaryNodeOp { const std::string type() { return "ReLU"; } }; +/** + * Represents a leaky + * rectified linear unit node in an expression graph. + * It is equivalent to the parametric ReLU with \f$ \alpha = 0.01 \f$. + * + * This node implements the activation function: + * \f[ + * f(x) = + * \begin{cases} + * 0.01 & \text{if } x < 0 \\ + * x & \text{if } x \geq 0 + * \end{cases} + * \f] + * + * and its derivative: + * \f[ + * f^\prime(x) = + * \begin{cases} + * 0.01 & \text{if } x < 0 \\ + * 1 & \text{if } x \geq 0 + * \end{cases} + * \f] + */ +struct LeakyReLUNodeOp : public UnaryNodeOp { + template + LeakyReLUNodeOp(Args... args) : UnaryNodeOp(args...) {} + + NodeOps forwardOps() { + return {NodeOp(Element(_1 = LeakyReLU(_2), val_, child(0)->val()))}; + } + + NodeOps backwardOps() { + return {NodeOp( + Add(_1 * LeakyReLUback(_2), child(0)->grad(), adj_, child(0)->val()))}; + } + + const std::string type() { return "LeakyReLU"; } +}; + /** * Represents a swish node * in an expression graph. -- cgit v1.2.3 From 857289d1bc0afa14d1338f7b6a29cb614b03bcac Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Sun, 29 Oct 2017 17:57:53 +0000 Subject: Add PReLU --- src/graph/node_operators_unary.h | 53 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 4 deletions(-) (limited to 'src/graph/node_operators_unary.h') diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 93c12a3c..819192dc 100644 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -243,8 +243,8 @@ struct ReLUNodeOp : public UnaryNodeOp { * \f[ * f(x) = * \begin{cases} - * 0.01 & \text{if } x < 0 \\ - * x & \text{if } x \geq 0 + * 0.01 & \text{if } x \leq 0 \\ + * x & \text{if } x > 0 * \end{cases} * \f] * @@ -252,8 +252,8 @@ struct ReLUNodeOp : public UnaryNodeOp { * \f[ * f^\prime(x) = * \begin{cases} - * 0.01 & \text{if } x < 0 \\ - * 1 & \text{if } x \geq 0 + * 0.01 & \text{if } x \leq 0 \\ + * 1 & \text{if } x > 0 * \end{cases} * \f] */ @@ -273,6 +273,51 @@ struct LeakyReLUNodeOp : public UnaryNodeOp { const std::string type() { return "LeakyReLU"; } }; +/** + * Represents a parametric + * rectified linear unit node in an expression graph. + * For \f$ \alpha = 0.01 \f$ (the default value) it is equivalent to Leaky + * ReLU. + * + * This node implements the activation function: + * \f[ + * f(x, \alpha) = + * \begin{cases} + * \alpha x & \text{if } x \leq 0 \\ + * x & \text{if } x > 0 + * \end{cases} + * \f] + * + * and its derivative: + * \f[ + * f^\prime(x, \alpha) = + * \begin{cases} + * \alpha & \text{if } x \leq 0 \\ + * 1 & \text{if } x > 0 + * \end{cases} + * \f] + */ +struct PReLUNodeOp : public UnaryNodeOp { + template + PReLUNodeOp(float alpha, Args... args) + : UnaryNodeOp(args...), alpha_(alpha) {} + + NodeOps forwardOps() { + return {NodeOp(Element(_1 = PReLU(_2, alpha_), val_, child(0)->val()))}; + } + + NodeOps backwardOps() { + return {NodeOp( + Add(_1 * PReLUback(_2, alpha_), child(0)->grad(), adj_, child(0)->val()))}; + } + + const std::string type() { return "PReLU"; } + +private: + float alpha_{0.01}; +}; + /** * Represents a swish node * in an expression graph. -- cgit v1.2.3 From 34e13035203987c856d6ba14f18d855f5f28cb19 Mon Sep 17 00:00:00 2001 From: Roman Grundkiewicz Date: Sun, 29 Oct 2017 18:22:26 +0000 Subject: Use PReLU in LeakyReLU --- src/graph/node_operators_unary.h | 48 ++++------------------------------------ 1 file changed, 4 insertions(+), 44 deletions(-) (limited to 'src/graph/node_operators_unary.h') diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 819192dc..7433c12a 100644 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -233,46 +233,6 @@ struct ReLUNodeOp : public UnaryNodeOp { const std::string type() { return "ReLU"; } }; -/** - * Represents a leaky - * rectified linear unit node in an expression graph. - * It is equivalent to the parametric ReLU with \f$ \alpha = 0.01 \f$. - * - * This node implements the activation function: - * \f[ - * f(x) = - * \begin{cases} - * 0.01 & \text{if } x \leq 0 \\ - * x & \text{if } x > 0 - * \end{cases} - * \f] - * - * and its derivative: - * \f[ - * f^\prime(x) = - * \begin{cases} - * 0.01 & \text{if } x \leq 0 \\ - * 1 & \text{if } x > 0 - * \end{cases} - * \f] - */ -struct LeakyReLUNodeOp : public UnaryNodeOp { - template - LeakyReLUNodeOp(Args... args) : UnaryNodeOp(args...) {} - - NodeOps forwardOps() { - return {NodeOp(Element(_1 = LeakyReLU(_2), val_, child(0)->val()))}; - } - - NodeOps backwardOps() { - return {NodeOp( - Add(_1 * LeakyReLUback(_2), child(0)->grad(), adj_, child(0)->val()))}; - } - - const std::string type() { return "LeakyReLU"; } -}; - /** * Represents a parametric @@ -308,8 +268,8 @@ struct PReLUNodeOp : public UnaryNodeOp { } NodeOps backwardOps() { - return {NodeOp( - Add(_1 * PReLUback(_2, alpha_), child(0)->grad(), adj_, child(0)->val()))}; + return {NodeOp(Add( + _1 * PReLUback(_2, alpha_), child(0)->grad(), adj_, child(0)->val()))}; } const std::string type() { return "PReLU"; } @@ -809,8 +769,8 @@ struct TransposeNodeOp : public UnaryNodeOp { Shape newShape(Expr a, Shape permute) { Shape shape = a->shape(); - UTIL_THROW_IF2(shape.size() != permute.size(), - "Shape and transpose axis have different number of dimensions"); + ABORT_IF(shape.size() != permute.size(), + "Shape and transpose axis have different number of dimensions"); for(int i = 0; i < shape.size(); ++i) shape.set(i, a->shape()[permute[i]]); -- cgit v1.2.3