Merge pull request #1081 from huihuifan/master

adding gated linear unit and tests to nn

4 files changed, 96 insertions, 0 deletions

diff --git a/GatedLinearUnit.lua b/GatedLinearUnit.lua
new file mode 100644
index 0000000..5f215ca
--- /dev/null
+++ b/GatedLinearUnit.lua
@@ -0,0 +1,42 @@
+local GatedLinearUnit, parent = torch.class('nn.GatedLinearUnit', 'nn.Module')
+
+function GatedLinearUnit:__init(dim)
+   parent.__init(self)
+   self.sigmoid = nn.Sigmoid()
+   self.dim = dim
+end
+
+function GatedLinearUnit:updateOutput(input)
+    local dim = self.dim or input:dim()
+    local inputSize = input:size(dim)
+
+    assert(inputSize % 2 == 0, "halving dimension needs to be even")
+
+    self.fHalf = input:narrow(dim, 1, inputSize/2)
+    self.sHalf = input:narrow(dim, inputSize/2 + 1, inputSize/2)
+
+    self.sHalfOut = self.sigmoid:forward(self.sHalf)
+    self.output:resizeAs(self.fHalf):copy(self.fHalf):cmul(self.sHalfOut)
+
+    return self.output
+end
+
+function GatedLinearUnit:updateGradInput(input, gradOutput)
+    local dim = self.dim or input:dim()
+    local inputSize = input:size(dim)
+
+    assert(inputSize % 2 == 0, "halving dimension needs to be even")
+
+    local fGradInput = self.sHalfOut
+    local sGradInput = self.sigmoid:backward(self.sHalf, gradOutput)
+                                   :cmul(self.fHalf)
+
+    self.gradInput:resizeAs(input)
+    self.gradInput:narrow(dim, 1, inputSize/2)
+                    :copy(fGradInput)
+                    :cmul(gradOutput)
+    self.gradInput:narrow(dim, inputSize/2+1, inputSize/2)
+                    :copy(sGradInput)
+
+    return self.gradInput
+end
diff --git a/doc/transfer.md b/doc/transfer.md
index 3d2d034..814aedf 100644
--- a/doc/transfer.md
+++ b/doc/transfer.md
@@ -522,6 +522,16 @@ Can optionally do its operation in-place without using extra state memory:
 f = nn.LeakyReLU(negval, true) -- true = in-place, false = keeping separate state.
 ```
 
+<a name="nn.GatedLinearUnit"></a>
+## GatedLinearUnit ##
+
+Applies a Gated Linear unit activation function, which halves the input dimension as follows:
+
+`GatedLinearUnit` is defined as `f([x1, x2])` = `x1 * sigmoid(x2)`
+
+where x1 is the first half of the input vector and x2 is the second half. The multiplication is component-wise, and the input vector must have an even number of elements.
+
+The GatedLinearUnit optionally takes a `dim` parameter, which is the dimension of the input tensor to operate over. It defaults to the last dimension.
 
 <a name="nn.SpatialSoftMax"></a>
 ## SpatialSoftMax ##
diff --git a/init.lua b/init.lua
index 1e3924b..cb9cf26 100644
--- a/init.lua
+++ b/init.lua
@@ -94,6 +94,7 @@ require('nn.SpatialSoftMax')
 require('nn.SpatialLogSoftMax')
 require('nn.RReLU')
 require('nn.ELU')
+require('nn.GatedLinearUnit')
 
 require('nn.LookupTable')
 require('nn.SpatialConvolution')
diff --git a/test.lua b/test.lua
index 5cb7066..56ef234 100644
--- a/test.lua
+++ b/test.lua
@@ -562,6 +562,49 @@ function nntest.ReLU6()
    end
 end
 
+function nntest.GatedLinearUnit()
+   local model = nn.GatedLinearUnit()
+   local t = torch.Tensor({{1, 1}, {2, 2}, {3, 3}})
+   local thalf = torch.Tensor():resizeAs(t):copy(t):narrow(2, 1, 1)
+   mytester:assertTensorEq(
+      thalf:cmul(torch.sigmoid(thalf)),
+      model:forward(t):resizeAs(thalf),
+      0.000001,
+      'Gated Linear output'
+   )
+   t = torch.Tensor({{1, 1, 1, 1}, {2, 2, 2, 2}, {3, 3, 3, 3}})
+   thalf = torch.Tensor():resizeAs(t):copy(t):narrow(2, 1, 2)
+   mytester:assertTensorEq(
+      thalf:cmul(torch.sigmoid(thalf)),
+      model:forward(t),
+      0.000001,
+      'Gated Linear Unit output'
+   )
+
+   local input = torch.rand(1, 10)
+   local err = jac.testJacobian(model, input)
+   mytester:assert(err < precision, 'Gated Linear gradient')
+
+   input = torch.rand(5, 10, 6)
+   model = nn.GatedLinearUnit(2)
+   err = jac.testJacobian(model, input)
+   mytester:assert(err < precision, 'Gated Linear gradient, non-default dim')
+
+   input = torch.rand(5, 10, 6)
+   model = nn.GatedLinearUnit(3)
+   err = jac.testJacobian(model, input)
+   mytester:assert(err < precision, 'Gated Linear gradient, non-default dim')
+
+   input = torch.rand(5, 10)
+   model = nn.Sequential()
+   model:add(nn.Linear(10, 10))
+   model:add(nn.GatedLinearUnit())
+   model:add(nn.ReLU())
+   model:add(nn.LogSoftMax())
+   err = jac.testJacobian(model, input)
+   mytester:assert(err < precision, 'Gated Linear gradient with other layers')
+end
+
 function nntest.Exp()
    local ini = math.random(3,5)
    local inj = math.random(3,5)