diff options
author | Roman Grundkiewicz <rgrundki@exseed.ed.ac.uk> | 2017-10-29 15:39:23 +0300 |
---|---|---|
committer | Roman Grundkiewicz <rgrundki@exseed.ed.ac.uk> | 2017-10-29 20:26:35 +0300 |
commit | 3d226d9bd526694ffecb39c6ba15cd2748ccf7ab (patch) | |
tree | 397255f9f8e79befa84fa0f2a64d9d9b1f4565ad /src/graph/node_operators_unary.h | |
parent | a9279f1c91f2cc5cf54d036f52a9b9bc14698408 (diff) |
Add a comment for SwishNodeOp
Diffstat (limited to 'src/graph/node_operators_unary.h')
-rw-r--r-- | src/graph/node_operators_unary.h | 34 |
1 files changed, 21 insertions, 13 deletions
diff --git a/src/graph/node_operators_unary.h b/src/graph/node_operators_unary.h index 2c205f17..38241258 100644 --- a/src/graph/node_operators_unary.h +++ b/src/graph/node_operators_unary.h @@ -196,21 +196,19 @@ struct TanhNodeOp : public NaryNodeOp { /** * Represents a <a -href="https://en.wikipedia.org/wiki/Rectifier_(neural_networks)">rectified -linear</a> node - * in an expression graph. + * href="https://en.wikipedia.org/wiki/Rectifier_(neural_networks)">rectified + * linear</a> node in an expression graph. * - * This node implements the <a -href="https://en.wikipedia.org/wiki/Activation_function">activation function</a> - * \f$f(x) = \max(0, x)\f$ and its derivative: + * This node implements the activationfunction \f$ f(x) = \max(0, x) \f$ and + * its derivative: * - \f[ - f^\prime(x) = - \begin{cases} - 0 & \text{if } x \leq 0 \\ - 1 & \text{if } x > 0 - \end{cases} -\f] + * \f[ + * f^\prime(x) = + * \begin{cases} + * 0 & \text{if } x \leq 0 \\ + * 1 & \text{if } x > 0 + * \end{cases} + * \f] */ struct ReLUNodeOp : public UnaryNodeOp { template <typename... Args> @@ -228,6 +226,16 @@ struct ReLUNodeOp : public UnaryNodeOp { const std::string type() { return "ReLU"; } }; +/** + * Represents a <a href="https://arxiv.org/pdf/1710.05941.pdf">swish</a> node + * in an expression graph. + * + * This node implements the activation function + * \f$ f(x) = x \cdot \sigma(x) \f$ + * and its derivative + * \f$ f^\prime(x) = f(x) + \sigma(x)(1 - f(x)) \f$ . + * + */ struct SwishNodeOp : public UnaryNodeOp { template <typename... Args> SwishNodeOp(Args... args) : UnaryNodeOp(args...) {} |