New NN classes

extra/nn/L1Cost.lua                       : L1 penalty
extra/nn/SpatialFullConvolution.lua       : full convolution
extra/nn/SpatialFullConvolutionMap.lua    : full convolution with connection table
extra/nn/TanhShrink.lua                   : shrinkage with x-tanh(x)
extra/nn/WeightedMSECriterion.lua         : mean squared error with weighting mask on the target

Add new nn classes that are used commonly for unsupervised training of convolutional auto encoders

1 files changed, 196 insertions, 2 deletions

diff --git a/test/test.lua b/test/test.lua
index 3097431..58a9bd7 100644
--- a/test/test.lua
+++ b/test/test.lua
@@ -7,7 +7,6 @@ local precision = 1e-5
 local expprecision = 1e-4
 
 local nntest = {}
-local nntestx = {}
 
 function nntest.Add()
    local ini = math.random(10,20)
@@ -341,6 +340,22 @@ function nntest.WeightedEuclidean()
    mytester:asserteq(berr, 0, torch.typename(module) .. ' - i/o backward err ')
 end
 
+function nntest.WeightedMSECriterion()
+   local from  = math.random(100,200)
+   local input = torch.Tensor(from):zero()
+   local target = torch.randn(from)
+   local weight = torch.randn(from)
+   local cri = nn.WeightedMSECriterion(weight)
+   local module = nn.CriterionModule(cri,target)
+
+   local err = jac.testJacobian(module, input)
+   mytester:assertlt(err, precision, 'error on state ')
+   
+   local ferr, berr = jac.testIO(module, input)
+   mytester:asserteq(0, ferr, torch.typename(module) .. ' - i/o forward err ')
+   mytester:asserteq(0, berr, torch.typename(module) .. ' - i/o backward err ')
+end
+
 function nntest.LogSigmoid()
    local ini = math.random(10,20)
    local inj = math.random(10,20)
@@ -795,6 +810,167 @@ function nntest.SpatialConvolutionMap()
    mytester:asserteq(0, berr, torch.typename(module) .. ' - i/o backward err ')
 end
 
+
+function nntest.SpatialFullConvolution()
+   local from = math.random(1,10)
+   local to = math.random(1,10)
+   local ki = math.random(1,10)
+   local kj = math.random(1,10)
+   local si = math.random(1,4)
+   local sj = math.random(1,4)
+   local ini = math.random(10,18)
+   local inj = math.random(10,18)
+   local module = nn.SpatialFullConvolution(from, to, ki, kj, si, sj)
+   local input = torch.Tensor(from, inj, ini):zero()
+   
+   -- stochastic
+      local err = jac.testJacobian(module, input)
+   mytester:assertlt(err, precision, 'error on state ')
+   
+   local err = jac.testJacobianParameters(module, input, module.weight, module.gradWeight)
+   mytester:assertlt(err , precision, 'error on weight ')
+   
+   local err = jac.testJacobianParameters(module, input, module.bias, module.gradBias)
+   mytester:assertlt(err , precision, 'error on bias ')
+
+   local err = jac.testJacobianUpdateParameters(module, input, module.weight)
+   mytester:assertlt(err , precision, 'error on weight [direct update] ')
+   
+   local err = jac.testJacobianUpdateParameters(module, input, module.bias)
+   mytester:assertlt(err , precision, 'error on bias [direct update] ')
+
+   for t,err in pairs(jac.testAllUpdate(module, input, 'weight', 'gradWeight')) do
+      mytester:assertlt(err, precision, string.format(
+                         'error on weight [%s]', t))
+   end
+
+   for t,err in pairs(jac.testAllUpdate(module, input, 'bias', 'gradBias')) do
+      mytester:assertlt(err, precision, string.format(
+                         'error on bias [%s]', t))
+   end
+
+   -- batch
+   local batch = math.random(2,5)
+   ini = math.random(4,8)
+   inj = math.random(4,8)
+   module = nn.SpatialFullConvolution(from, to, ki, kj, si, sj)
+   input = torch.Tensor(batch,from,inj,ini):zero()
+
+   local err = jac.testJacobian(module, input)
+   mytester:assertlt(err, precision, 'batch error on state ')
+   
+   local err = jac.testJacobianParameters(module, input, module.weight, module.gradWeight)
+   mytester:assertlt(err , precision, 'batch error on weight ')
+   
+   local err = jac.testJacobianParameters(module, input, module.bias, module.gradBias)
+   mytester:assertlt(err , precision, 'batch error on bias ')
+
+   local err = jac.testJacobianUpdateParameters(module, input, module.weight)
+   mytester:assertlt(err , precision, 'batch error on weight [direct update] ')
+   
+   local err = jac.testJacobianUpdateParameters(module, input, module.bias)
+   mytester:assertlt(err , precision, 'batch error on bias [direct update] ')
+
+   for t,err in pairs(jac.testAllUpdate(module, input, 'weight', 'gradWeight')) do
+      mytester:assertlt(err, precision, string.format(
+                         'error on weight [%s]', t))
+   end
+
+   for t,err in pairs(jac.testAllUpdate(module, input, 'bias', 'gradBias')) do
+      mytester:assertlt(err, precision, string.format(
+                         'batch error on bias [%s]', t))
+   end
+   
+   local ferr, berr = jac.testIO(module, input)
+   mytester:asserteq(0, ferr, torch.typename(module) .. ' - i/o forward err ')
+   mytester:asserteq(0, berr, torch.typename(module) .. ' - i/o backward err ')
+end
+
+function nntest.SpatialFullConvolutionMap()
+   local from = math.ceil(torch.uniform(2,5))
+   local to = math.ceil(torch.uniform(2,7))
+   local fanin = math.ceil(torch.uniform(1, from))
+   local tt = nn.tables.random(from, to, fanin)
+   local ki = math.ceil(torch.uniform(2,7))
+   local kj = math.ceil(torch.uniform(2,7))
+   local si = math.ceil(torch.uniform(1,3))
+   local sj = math.ceil(torch.uniform(1,3))
+   local ini = math.ceil(torch.uniform(10,18))
+   local inj = math.ceil(torch.uniform(10,18))
+   local module = nn.SpatialFullConvolutionMap(tt, ki, kj, si, sj)
+   local input = torch.Tensor(from, inj, ini):zero()
+   
+   -- stochastic
+      local err = jac.testJacobian(module, input)
+   mytester:assertlt(err, precision, 'error on state ')
+   
+   local err = jac.testJacobianParameters(module, input, module.weight, module.gradWeight)
+   mytester:assertlt(err , precision, 'error on weight ')
+   
+   local err = jac.testJacobianParameters(module, input, module.bias, module.gradBias)
+   mytester:assertlt(err , precision, 'error on bias ')
+
+   local err = jac.testJacobianUpdateParameters(module, input, module.weight)
+   mytester:assertlt(err , precision, 'error on weight [direct update] ')
+   
+   local err = jac.testJacobianUpdateParameters(module, input, module.bias)
+   mytester:assertlt(err , precision, 'error on bias [direct update] ')
+
+   for t,err in pairs(jac.testAllUpdate(module, input, 'weight', 'gradWeight')) do
+      mytester:assertlt(err, precision, string.format(
+                         'error on weight [%s]', t))
+   end
+
+   for t,err in pairs(jac.testAllUpdate(module, input, 'bias', 'gradBias')) do
+      mytester:assertlt(err, precision, string.format(
+                         'error on bias [%s]', t))
+   end
+   
+   local ferr, berr = jac.testIO(module, input)
+   mytester:asserteq(0, ferr, torch.typename(module) .. ' - i/o forward err ')
+   mytester:asserteq(0, berr, torch.typename(module) .. ' - i/o backward err ')
+end
+
+function nntest.SpatialFullConvolutionCompare()
+    local from = math.ceil(torch.uniform(2,5))
+    local to = math.ceil(torch.uniform(2,7))
+    local tt = nn.tables.full(from, to)
+    local ki = math.ceil(torch.uniform(2,7))
+    local kj = math.ceil(torch.uniform(2,7))
+    local si = math.ceil(torch.uniform(1,3))
+    local sj = math.ceil(torch.uniform(1,3))
+    local ini = math.ceil(torch.uniform(10,18))
+    local inj = math.ceil(torch.uniform(10,18))
+    local module1 = nn.SpatialFullConvolutionMap(tt, ki, kj, si, sj)
+    local module2 = nn.SpatialFullConvolution(from, to, ki, kj, si, sj)
+    local input = torch.rand(from, inj, ini)
+    for k=1,tt:size(1) do
+       module1.weight[k]:copy(module2.weight[tt[k][1]][tt[k][2]])
+       module1.bias:copy(module2.bias)
+    end
+
+    local o1 = module1:updateOutput(input)
+    local o2 = module2:updateOutput(input)
+    mytester:assertlt(o1:dist(o2), precision, 'error on output')
+
+    local go1 = torch.rand(o1:size())
+    local go2 = go1:clone()
+
+    local gi1= module1:updateGradInput(input,go1)
+    local gi2 = module2:updateGradInput(input,go2)
+    mytester:assertlt(gi1:dist(gi2), precision, 'error on gradInput')
+
+    module1:zeroGradParameters()
+    module2:zeroGradParameters()
+
+    module1:accGradParameters(input,go1)
+    module2:accGradParameters(input,go2)
+    for k=1,tt:size(1) do
+      mytester:assertlt(module1.gradWeight[k]:dist(module2.gradWeight[tt[k][1]][tt[k][2]]),precision,'error on gradWeight ' .. k)
+    end
+    mytester:assertlt(module1.gradBias:dist(module2.gradBias),precision,'error on gradBias ')
+end
+
 local function batchcompare(smod, sin, plist)
    local bs = torch.LongStorage(sin:size():size()+1)
    bs[1] = 1
@@ -841,6 +1017,24 @@ function nntest.SpatialConvolutionBatchCompare()
    batchcompare(module,input, {'weight','bias','gradWeight','gradBias'})
 end
 
+function nntest.SpatialFullConvolutionBatchCompare()
+   local from = math.random(1,10)
+   local to = math.random(1,10)
+   local ki = math.random(1,10)
+   local kj = math.random(1,10)
+   local si = math.random(1,4)
+   local sj = math.random(1,4)
+   local ini = math.random(10,18)
+   local inj = math.random(10,18)
+
+   local module = nn.SpatialFullConvolution(from, to, ki, kj, si, sj)
+   local input = torch.randn(from, inj, ini)
+
+   batchcompare(module,input, {'weight','bias','gradWeight','gradBias'})
+end
+   
+
+
 function nntest.SpatialSubSamplingBatchCompare()
    local from = math.random(1,10)
    local ki = math.random(1,5)
@@ -1105,7 +1299,7 @@ function nntest.TemporalSubSampling()
    mytester:asserteq(0, berr, torch.typename(module) .. ' - i/o backward err ')
 end
 
-function nntestx.TemporalMaxPooling()
+function nntest.TemporalMaxPooling()
    local from = math.random(1,10)
    local ki = math.random(1,10)
    local si = math.random(1,4)