Add extra tests for SparseLinear and fix bug where scale was not being multiplied into bias updates

3 files changed, 88 insertions, 8 deletions

diff --git a/SparseJacobian.lua b/SparseJacobian.lua
index 388b8bf..3dd489d 100644
--- a/SparseJacobian.lua
+++ b/SparseJacobian.lua
@@ -35,6 +35,44 @@ function nn.SparseJacobian.backward (module, input, param, dparam)
    return jacobian
 end
 
+
+function nn.SparseJacobian.backwardUpdate (module, input, param)
+
+   -- output deriv
+   module:forward(input)
+   local dout = module.output.new():resizeAs(module.output)
+   -- 1D view
+   local sdout = module.output.new(dout:storage(),1,dout:nElement())
+   -- jacobian matrix to calculate
+   local jacobian = torch.Tensor(param:nElement(),dout:nElement()):zero()
+
+   -- original param
+   local params = module:parameters()
+   local origparams = {}
+   for j=1,#params do
+      table.insert(origparams, params[j]:clone())
+   end
+
+   for i=1,sdout:nElement() do
+      -- Reset parameters
+      for j=1,#params do
+         params[j]:copy(origparams[j])
+      end
+      dout:zero()
+      sdout[i] = 1
+      module:zeroGradParameters()
+      local din = module:updateGradInput(input, dout)
+      module:accUpdateGradParameters(input, dout, 1)
+      jacobian:select(2,i):copy(param)
+   end
+
+   for j=1,#params do
+      params[j]:copy(origparams[j])
+   end
+
+   return jacobian
+end
+
 function nn.SparseJacobian.forward(module, input, param)
    local doparam = 0
    if param then
@@ -81,6 +119,33 @@ function nn.SparseJacobian.forward(module, input, param)
    return jacobian
 end
 
+function nn.SparseJacobian.forwardUpdate(module, input, param)
+   -- perturbation amount
+   local small = 1e-6
+   -- 1D view of input
+   --local tst = param:storage()
+   local sin =  param.new(param):resize(param:nElement())--param.new(tst,1,tst:size())
+   -- jacobian matrix to calculate
+   local jacobian = torch.Tensor():resize(param:nElement(),module:forward(input):nElement())
+   
+   local outa = torch.Tensor(jacobian:size(2))
+   local outb = torch.Tensor(jacobian:size(2))
+   
+   for i=1,sin:nElement() do      
+      sin[i] = sin[i] - small
+      outa:copy(module:forward(input))
+      sin[i] = sin[i] + 2*small
+      outb:copy(module:forward(input))
+      sin[i] = sin[i] - small
+
+      outb:add(-1,outa):div(2*small)
+      jacobian:select(1,i):copy(outb)
+      jacobian:select(1,i):mul(-1)
+      jacobian:select(1,i):add(sin[i])
+   end
+   return jacobian
+end
+
 function nn.SparseJacobian.testJacobian (module, input, minval, maxval)
    minval = minval or -2
    maxval = maxval or 2
@@ -104,6 +169,19 @@ function nn.SparseJacobian.testJacobianParameters (module, input, param, dparam,
    return error:abs():max()
 end
 
+function nn.SparseJacobian.testJacobianUpdateParameters (module, input, param, minval, maxval)
+   minval = minval or -2
+   maxval = maxval or 2
+   local inrange = maxval - minval
+   input:select(2,2):copy(torch.rand(input:size(1)):mul(inrange):add(minval))
+   param:copy(torch.rand(param:nElement()):mul(inrange):add(minval))
+   local params_bprop = nn.SparseJacobian.backwardUpdate(module, input, param)
+   local params_fprop = nn.SparseJacobian.forwardUpdate(module, input, param)
+
+   local error = params_fprop - params_bprop
+   return error:abs():max()
+end
+
 function nn.SparseJacobian.testIO(module,input, minval, maxval)
    minval = minval or -2
    maxval = maxval or 2
diff --git a/generic/SparseLinear.c b/generic/SparseLinear.c
index a7d0e36..5e0143f 100644
--- a/generic/SparseLinear.c
+++ b/generic/SparseLinear.c
@@ -53,10 +53,6 @@ static int nn_(SparseLinear_accGradParameters)(lua_State *L)
       if(offset >= 0 && offset < dim) /* make sure indices are in bounds.. */
       {
           real val = scale*THTensor_(get2d)(input, i, 1);
-          THBlas_(scal)(gradOutput->size[0],
-                        0, 
-                        THTensor_(data)(gradWeight)+offset*gradWeight->stride[1],
-                        gradWeight->stride[0]); /* zero */
           
           THBlas_(axpy)(gradOutput->size[0], 
                         val, 
@@ -71,7 +67,7 @@ static int nn_(SparseLinear_accGradParameters)(lua_State *L)
       }
   }
   
-  THTensor_(cadd)(gradBias, gradBias, 1, gradOutput); 
+  THTensor_(cadd)(gradBias, gradBias, scale, gradOutput); 
   
   if(weightDecay != 0)
     THTensor_(cadd)(gradWeight, gradWeight, weightDecay, weight);
diff --git a/test/test.lua b/test/test.lua
index 27bb114..eb6dede 100644
--- a/test/test.lua
+++ b/test/test.lua
@@ -238,8 +238,8 @@ function nntest.Sqrt()
 end
 
 function nntest.Linear()
-   local ini = math.random(50,70)
-   local inj = math.random(50,70)
+   local ini = math.random(5,7)
+   local inj = math.random(5,7)
    local input = torch.Tensor(ini):zero()
    local module = nn.Linear(ini,inj)
 
@@ -335,7 +335,7 @@ function nntest.SparseLinear()
    local err = (expected - actual):abs():max()
    mytester:assertle(err, precision, 'error on result')
 
-   -- Jacobian
+   -- Jacobian 1D
    local err = sjac.testJacobian(module,input)
    mytester:assertlt(err,precision, 'error on state ')
 
@@ -345,6 +345,12 @@ function nntest.SparseLinear()
    local err = sjac.testJacobianParameters(module, input, module.bias, module.gradBias)
    mytester:assertlt(err,precision, 'error on bias ')
    
+   local err = sjac.testJacobianUpdateParameters(module, input, module.weight)
+   mytester:assertlt(err,precision, 'error on weight [direct update] ')
+
+   local err = sjac.testJacobianUpdateParameters(module, input, module.bias)
+   mytester:assertlt(err,precision, 'error on bias [direct update] ')
+
    local ferr, berr = sjac.testIO(module, input)
    mytester:asserteq(0, ferr, torch.typename(module) .. ' - i/o forward err ')
    mytester:asserteq(0, berr, torch.typename(module) .. ' - i/o backward err ')