diff options
| -rw-r--r-- | SparseJacobian.lua | 141 | ||||
| -rw-r--r-- | SparseLinear.lua | 17 | ||||
| -rw-r--r-- | generic/SparseLinear.c | 101 | ||||
| -rw-r--r-- | init.lua | 1 | ||||
| -rw-r--r-- | test/test.lua | 50 |
5 files changed, 261 insertions, 49 deletions
diff --git a/SparseJacobian.lua b/SparseJacobian.lua new file mode 100644 index 0000000..388b8bf --- /dev/null +++ b/SparseJacobian.lua @@ -0,0 +1,141 @@ +nn.SparseJacobian = {} + +function nn.SparseJacobian.backward (module, input, param, dparam) + local doparam = 0 + if param then + doparam = 1 + end + + -- output deriv + module:forward(input) + local dout = module.output.new():resizeAs(module.output) + -- 1D view + local sdout = module.output.new(dout:storage(), 1, dout:nElement()) + -- jacobian matrix to calculate + local jacobian + if doparam == 1 then + jacobian = torch.Tensor(param:nElement(), dout:nElement()):zero() + else + jacobian = torch.Tensor(input:size(1), dout:nElement()):zero() + end + + for i=1,sdout:nElement() do + dout:zero() + sdout[i] = 1 + module:zeroGradParameters() + local din = module:updateGradInput(input, dout) + module:accGradParameters(input, dout) + if doparam == 1 then + jacobian:select(2,i):copy(dparam) + else + jacobian:select(2,i):copy(din:select(2,2)) + end + end + + return jacobian +end + +function nn.SparseJacobian.forward(module, input, param) + local doparam = 0 + if param then + doparam = 1 + end + param = param or input + + -- perturbation amount + local small = 1e-6 + -- 1D view of input + --local tst = param:storage() + local sin + if doparam == 1 then + sin = param.new(param):resize(param:nElement()) + else + sin = input.new(input):select(2,2) + end + + local out = module:forward(input) + -- jacobian matrix to calculate + local jacobian + if doparam == 1 then + jacobian = torch.Tensor():resize(param:nElement(), + out:nElement()) + else + jacobian = torch.Tensor():resize(input:size(1), + out:nElement()) + end + + local outa = torch.Tensor(jacobian:size(2)) + local outb = torch.Tensor(jacobian:size(2)) + + for i=1,sin:nElement() do + sin[i] = sin[i] - small + outa:copy(module:forward(input)) + sin[i] = sin[i] + 2*small + outb:copy(module:forward(input)) + sin[i] = sin[i] - small + + outb:add(-1,outa):div(2*small) + jacobian:select(1,i):copy(outb) + end + + return jacobian +end + +function nn.SparseJacobian.testJacobian (module, input, minval, maxval) + minval = minval or -2 + maxval = maxval or 2 + local inrange = maxval - minval + input:select(2,2):copy(torch.rand(input:size(1)):mul(inrange):add(minval)) + local jac_fprop = nn.SparseJacobian.forward(module,input) + local jac_bprop = nn.SparseJacobian.backward(module,input) + local error = jac_fprop-jac_bprop + return error:abs():max() +end + +function nn.SparseJacobian.testJacobianParameters (module, input, param, dparam, minval, maxval) + minval = minval or -2 + maxval = maxval or 2 + local inrange = maxval - minval + input:select(2,2):copy(torch.rand(input:size(1)):mul(inrange):add(minval)) + param:copy(torch.rand(param:nElement()):mul(inrange):add(minval)) + local jac_bprop = nn.SparseJacobian.backward(module, input, param, dparam) + local jac_fprop = nn.SparseJacobian.forward(module, input, param) + local error = jac_fprop - jac_bprop + return error:abs():max() +end + +function nn.SparseJacobian.testIO(module,input, minval, maxval) + minval = minval or -2 + maxval = maxval or 2 + local inrange = maxval - minval + + -- run module + module:forward(input) + local go = module.output:clone():copy(torch.rand(module.output:nElement()):mul(inrange):add(minval)) + module:zeroGradParameters() + module:updateGradInput(input,go) + module:accGradParameters(input,go) + + local fo = module.output:clone() + local bo = module.gradInput:clone() + + -- write module + local f = torch.DiskFile('tmp.bin','w'):binary() + f:writeObject(module) + f:close() + -- read module + local m = torch.DiskFile('tmp.bin'):binary():readObject() + m:forward(input) + m:zeroGradParameters() + m:updateGradInput(input,go) + m:accGradParameters(input,go) + -- cleanup + os.remove('tmp.bin') + + local fo2 = m.output:clone() + local bo2 = m.gradInput:clone() + + local errf = fo - fo2 + local errb = bo - bo2 + return errf:abs():max(), errb:abs():max() +end diff --git a/SparseLinear.lua b/SparseLinear.lua index f1a2be5..735d0ed 100644 --- a/SparseLinear.lua +++ b/SparseLinear.lua @@ -42,3 +42,20 @@ end function SparseLinear:accGradParameters(input, gradOutput, scale) return input.nn.SparseLinear_accGradParameters(self, input, gradOutput, scale) end + +function SparseLinear:updateGradInput(input, gradOutput) + if self.gradInput then + self.gradInput:resize(input:size()) + self.gradInput:copy(input) + local numNonzero = self.gradInput:size(1) + for e=1,numNonzero do + local g = 0 + local i = self.gradInput[{e,1}] + for j=1,self.output:size(1) do + g = g + self.weight[{j,i}] * gradOutput[j] + end + self.gradInput[{e,2}] = g + end + return self.gradInput + end +end
\ No newline at end of file diff --git a/generic/SparseLinear.c b/generic/SparseLinear.c index c602c2a..a7d0e36 100644 --- a/generic/SparseLinear.c +++ b/generic/SparseLinear.c @@ -9,25 +9,26 @@ static int nn_(SparseLinear_updateOutput)(lua_State *L) THTensor * weight = luaT_getfieldcheckudata(L, 1, "weight", torch_Tensor); THTensor * bias = luaT_getfieldcheckudata(L, 1, "bias", torch_Tensor); THTensor * output = luaT_getfieldcheckudata(L, 1, "output", torch_Tensor); - long dim = weight->size[0]; /* number of weights.. */ + long dim = weight->size[1]; /* number of weights.. */ THTensor_(copy)(output, bias); - for(i = 0; i < input->size[1]; i++) + for(i = 0; i < input->size[0]; i++) { - long offset = (long)(THTensor_(get2d)(input, 0, i))-1; - + long offset = (long)(THTensor_(get2d)(input, i, 0)) - 1; if(offset >= 0 && offset < dim) /* make sure indices are in bounds.. */ { - real val = THTensor_(get2d)(input, 1, i); - THBlas_(axpy)(output->size[0], - val, - THTensor_(data)(weight)+offset*weight->stride[0], - weight->stride[1], - THTensor_(data)(output), - output->stride[0]); + real val = THTensor_(get2d)(input, i, 1); + THBlas_(axpy)(output->size[0], + val, + THTensor_(data)(weight)+offset*weight->stride[1], + weight->stride[0], + THTensor_(data)(output), + output->stride[0]); + } + else { + printf("\nOutput: %d not between 0 and %d\n", offset, dim-1); + luaL_error(L, "index out of bound"); } - else - luaL_error(L, "index out of bound"); } return 1; } @@ -43,29 +44,31 @@ static int nn_(SparseLinear_accGradParameters)(lua_State *L) THTensor * gradWeight = luaT_getfieldcheckudata(L, 1, "gradWeight", torch_Tensor); THTensor * lastInput = luaT_getfieldcheckudata(L, 1, "lastInput", torch_Tensor); real weightDecay = luaT_getfieldchecknumber(L, 1, "weightDecay"); - long dim = gradWeight->size[0]; /* number of weights.. */ + long dim = gradWeight->size[1]; /* number of weights.. */ - for(i = 0; i < input->size[1]; i++) + for(i = 0; i < input->size[0]; i++) { - long offset = (long)(THTensor_(get2d)(input, 0, i))-1; - - if(offset >= 0 && offset < dim) /* make sure indices are in bounds.. */ - { - real val = scale*THTensor_(get2d)(input, 1, i); - THBlas_(scal)(gradOutput->size[0], - 0, - THTensor_(data)(gradWeight)+offset*gradWeight->stride[0], - gradWeight->stride[1]); /* zero */ + long offset = (long)(THTensor_(get2d)(input, i, 0)) - 1; - THBlas_(axpy)(gradOutput->size[0], - val, - THTensor_(data)(gradOutput), - gradOutput->stride[0], - THTensor_(data)(gradWeight)+offset*gradWeight->stride[0], - gradWeight->stride[1]); - } - else - luaL_error(L, "index out of bound"); + if(offset >= 0 && offset < dim) /* make sure indices are in bounds.. */ + { + real val = scale*THTensor_(get2d)(input, i, 1); + THBlas_(scal)(gradOutput->size[0], + 0, + THTensor_(data)(gradWeight)+offset*gradWeight->stride[1], + gradWeight->stride[0]); /* zero */ + + THBlas_(axpy)(gradOutput->size[0], + val, + THTensor_(data)(gradOutput), + gradOutput->stride[0], + THTensor_(data)(gradWeight)+offset*gradWeight->stride[1], + gradWeight->stride[0]); + } + else { + printf("\nAccG: %d not between 0 and %d\n", offset, dim-1); + luaL_error(L, "index out of bound"); + } } THTensor_(cadd)(gradBias, gradBias, 1, gradOutput); @@ -89,24 +92,26 @@ int nn_(SparseLinear_updateParameters)(lua_State *L) THTensor * gradWeight = luaT_getfieldcheckudata(L, 1, "gradWeight", torch_Tensor); THTensor * lastInput = luaT_getfieldcheckudata(L, 1, "lastInput", torch_Tensor); - long dim = weight->size[0]; /* number of weights.. */ + long dim = weight->size[1]; /* number of weights.. */ THTensor_(cadd)(bias, bias, -learningRate, gradBias); - for(i = 0; i < lastInput->size[1]; i++) + for(i = 0; i < lastInput->size[0]; i++) { - long offset = (long)(THTensor_(get2d)(lastInput, 0, i))-1; - - if(offset >= 0 && offset < dim) /* make sure indices are in bounds.. */ - { - THBlas_(axpy)(bias->size[0], - -learningRate, - THTensor_(data)(gradWeight)+offset*gradWeight->stride[0], - gradWeight->stride[1], - THTensor_(data)(weight)+offset*weight->stride[0], - weight->stride[1]); - } - else - luaL_error(L, "index out of bound"); + long offset = (long)(THTensor_(get2d)(lastInput, i, 0)) - 1; + + if(offset >= 0 && offset < dim) /* make sure indices are in bounds.. */ + { + THBlas_(axpy)(bias->size[0], + -learningRate, + THTensor_(data)(gradWeight)+offset*gradWeight->stride[1], + gradWeight->stride[0], + THTensor_(data)(weight)+offset*weight->stride[1], + weight->stride[0]); + } + else { + printf("\nUpdateP: %d not between 0 and %d\n", offset, dim-1); + luaL_error(L, "index out of bound"); + } } return 0; } @@ -103,5 +103,6 @@ include('WeightedMSECriterion.lua') include('StochasticGradient.lua') include('Jacobian.lua') +include('SparseJacobian.lua') include('hessian.lua') include('test.lua') diff --git a/test/test.lua b/test/test.lua index 0d54e3d..27bb114 100644 --- a/test/test.lua +++ b/test/test.lua @@ -2,6 +2,7 @@ require 'torch' local mytester = torch.Tester() local jac +local sjac local precision = 1e-5 local expprecision = 1e-4 @@ -303,6 +304,52 @@ function nntest.Linear() mytester:asserteq(berr, 0, torch.typename(module) .. ' - i/o backward err ') end +function nntest.SparseLinear() + local ini = math.random(5000,10000) + local inj = math.random(50,100) + local numNonzero = math.random(5,20) + + local module = nn.SparseLinear(ini,inj) + + -- Create a random sparse vector + N = {} + for i = 1, ini do N[i] = i end + for i = 1, numNonzero do + local j = math.random(i,ini) + N[i], N[j] = N[j], N[i] + end + local input = torch.Tensor(numNonzero, 2):zero() + for i = 1, numNonzero do input[{i,1}] = N[i] end + local values = input:select(2,2) + values:copy(torch.rand(values:nElement())):mul(2):add(-1) + + -- Check output + local actual = module:forward(input) + local expected = torch.Tensor(inj) + for j = 1, inj do + expected[j] = 0 + for i = 1,numNonzero do + expected[j] = expected[j] + values[i] * module.weight[{j, N[i]}] + end + end + local err = (expected - actual):abs():max() + mytester:assertle(err, precision, 'error on result') + + -- Jacobian + local err = sjac.testJacobian(module,input) + mytester:assertlt(err,precision, 'error on state ') + + local err = sjac.testJacobianParameters(module, input, module.weight, module.gradWeight) + mytester:assertlt(err,precision, 'error on weight ') + + local err = sjac.testJacobianParameters(module, input, module.bias, module.gradBias) + mytester:assertlt(err,precision, 'error on bias ') + + local ferr, berr = sjac.testIO(module, input) + mytester:asserteq(0, ferr, torch.typename(module) .. ' - i/o forward err ') + mytester:asserteq(0, berr, torch.typename(module) .. ' - i/o backward err ') +end + function nntest.Euclidean() local ini = math.random(50,70) local inj = math.random(50,70) @@ -347,7 +394,6 @@ end -- local weight = torch.randn(from) -- local cri = nn.WeightedMSECriterion(weight) -- local module = nn.CriterionModule(cri,target) - -- local err = jac.testJacobian(module, input) -- mytester:assertlt(err, precision, 'error on state ') @@ -1589,9 +1635,11 @@ mytester:add(nntest) if not nn then require 'nn' jac = nn.Jacobian + sjac = nn.SparseJacobian mytester:run() else jac = nn.Jacobian + sjac = nn.SparseJacobian function nn.test(tests) -- randomize stuff math.randomseed(os.time()) |