diff options
author | Ronan Collobert <ronan@collobert.com> | 2012-05-17 02:40:02 +0400 |
---|---|---|
committer | Ronan Collobert <ronan@collobert.com> | 2012-05-17 02:40:02 +0400 |
commit | 4ec6dec431c8523de7282ec3fdbbc6dd2d3cc243 (patch) | |
tree | f4bd0ed6a88944200cb6908d7186f47725888d44 | |
parent | 47f14c71716f8e38f310d29c819e355698a75377 (diff) | |
parent | 0ef82918f098b08e6a06b974c64b043bc461a62f (diff) |
Merge branch 'master' into mergeopenmp
Conflicts:
extra/openmp/pkg/openmp/generic/SpatialMaxPoolingOmp.c
extra/openmp/pkg/openmp/generic/SpatialSubSamplingOmp.c
extra/openmp/pkg/openmp/generic/SqrtOmp.c
extra/openmp/pkg/openmp/generic/SquareOmp.c
-rw-r--r-- | DistKLDivCriterion.lua | 64 | ||||
-rw-r--r-- | Log.lua | 20 | ||||
-rw-r--r-- | LookupTable.lua | 7 | ||||
-rw-r--r-- | Module.lua | 1 | ||||
-rw-r--r-- | Reshape.lua | 20 | ||||
-rw-r--r-- | SpatialContrastiveNormalization.lua | 42 | ||||
-rw-r--r-- | SpatialDivisiveNormalization.lua | 133 | ||||
-rw-r--r-- | dok/index.dok | 2 | ||||
-rw-r--r-- | init.lua | 4 | ||||
-rw-r--r-- | test/test.lua | 47 |
10 files changed, 328 insertions, 12 deletions
diff --git a/DistKLDivCriterion.lua b/DistKLDivCriterion.lua new file mode 100644 index 0000000..b6b6216 --- /dev/null +++ b/DistKLDivCriterion.lua @@ -0,0 +1,64 @@ +local DistKLDivCriterion, parent = torch.class('nn.DistKLDivCriterion', 'nn.Criterion') + +local epsilon = 1e-100 + +function DistKLDivCriterion:__init() + parent.__init(self) + self.sizeAverage = true +end + +function DistKLDivCriterion:updateOutput(input, target) + local log = math.log + if input:dim() == 1 then + self.output = 0 + for i = 1,input:size(1) do + local acc = 0 + if target[i] > 0 then + acc = target[i] * (log(target[i]) - input[i]) + end + self.output = self.output + acc + end + elseif input:dim() == 2 then + self.output = 0 + for i=1,target:size(1) do + local tar = target[i] + local inp = input[i] + for i = 1,inp:size(1) do + local acc = 0 + if tar[i] > epsilon then + acc = tar[i] * (log(tar[i]) - inp[i]) + end + self.output = self.output + acc + end + end + if self.sizeAverage then + self.output = self.output / target:size(1) + end + else + error('matrix or vector expected') + end + return self.output +end + +function DistKLDivCriterion:updateGradInput(input, target) + local gradInput = self.gradInput + gradInput:resizeAs(input) + + if input:dim() == 1 then + for i = 1,input:size(1) do + gradInput[i] = -target[i] + end + else + for i=1,target:size(1) do + local tar = target[i] + for i = 1,tar:size(1) do + gradInput[i] = -tar[i] + end + end + if self.sizeAverage then + gradInput:div(target:size(1)) + end + end + + return self.gradInput +end @@ -0,0 +1,20 @@ +local Log, parent = torch.class('nn.Log', 'nn.Module') + +function Log:__init(inputSize) + parent.__init(self) +end + +function Log:updateOutput(input) + self.output:resizeAs(input) + self.output:copy(input) + self.output:log() + return self.output +end + +function Log:updateGradInput(input, gradOutput) + self.gradInput:resizeAs(input) + self.gradInput:fill(1) + self.gradInput:cdiv(input) + self.gradInput:cmul(gradOutput) + return self.gradInput +end diff --git a/LookupTable.lua b/LookupTable.lua index 115f19c..7dfda7a 100644 --- a/LookupTable.lua +++ b/LookupTable.lua @@ -4,9 +4,10 @@ LookupTable.__version = 2 function LookupTable:__init(nIndex, ...) parent.__init(self) + local arg = {...} - if select('#', ...) == 1 and type(select(1, ...)) ~= "number" then - local size = select(1, ...) + if select('#', ...) == 1 and type(arg[1]) ~= "number" then + local size = arg[1] self.size = torch.LongStorage(#size + 1) for i=1,#size do self.size[i+1] = size[i] @@ -14,7 +15,7 @@ function LookupTable:__init(nIndex, ...) else self.size = torch.LongStorage(select('#', ...)+1) for i=1,select('#',...) do - self.size[i+1] = select(i, ...) + self.size[i+1] = arg[i] end end @@ -81,6 +81,7 @@ function Module:updateParameters(learningRate) end function Module:share(mlp, ...) + local arg = {...} for i,v in ipairs(arg) do if self[v] ~= nil then self[v]:set(mlp[v]) diff --git a/Reshape.lua b/Reshape.lua index 0be793f..753dd17 100644 --- a/Reshape.lua +++ b/Reshape.lua @@ -2,21 +2,25 @@ local Reshape, parent = torch.class('nn.Reshape', 'nn.Module') function Reshape:__init(...) parent.__init(self) + local arg = {...} + self.size = torch.LongStorage() self.batchsize = torch.LongStorage() - local n = select('#', ...) - if n == 1 and torch.typename(select(1, ...)) == 'torch.LongStorage' then - self.size:resize(#select(1, ...)):copy(select(1, ...)) + local n = #arg + if n == 1 and torch.typename(arg[1]) == 'torch.LongStorage' then + self.size:resize(#arg[1]):copy(arg[1]) else self.size:resize(n) - self.batchsize:resize(n+1) - self.nelement = 1 for i=1,n do - self.size[i] = select(i, ...) - self.batchsize[i+1] = select(i, ...) - self.nelement = self.nelement * self.size[i] + self.size[i] = arg[i] end end + self.nelement = 1 + self.batchsize:resize(#self.size+1) + for i=1,#self.size do + self.nelement = self.nelement * self.size[i] + self.batchsize[i+1] = self.size[i] + end end function Reshape:updateOutput(input) diff --git a/SpatialContrastiveNormalization.lua b/SpatialContrastiveNormalization.lua new file mode 100644 index 0000000..262d3b1 --- /dev/null +++ b/SpatialContrastiveNormalization.lua @@ -0,0 +1,42 @@ +local SpatialContrastiveNormalization, parent = torch.class('nn.SpatialContrastiveNormalization','nn.Module') + +function SpatialContrastiveNormalization:__init(nInputPlane, kernel, threshold, thresval) + parent.__init(self) + + -- get args + self.nInputPlane = nInputPlane or 1 + self.kernel = kernel or torch.Tensor(9,9):fill(1) + self.threshold = threshold or 1e-4 + self.thresval = thresval or 1e-4 + local kdim = self.kernel:nDimension() + + -- check args + if kdim ~= 2 and kdim ~= 1 then + error('<SpatialContrastiveNormalization> averaging kernel must be 2D or 1D') + end + if (self.kernel:size(1) % 2) == 0 or (kdim == 2 and (self.kernel:size(2) % 2) == 0) then + error('<SpatialContrastiveNormalization> averaging kernel must have ODD dimensions') + end + + -- instantiate sub+div normalization + self.normalizer = nn.Sequential() + self.normalizer:add(nn.SpatialSubtractiveNormalization(self.nInputPlane, self.kernel)) + self.normalizer:add(nn.SpatialDivisiveNormalization(self.nInputPlane, self.kernel, + self.threshold, self.threshval)) +end + +function SpatialContrastiveNormalization:updateOutput(input) + self.output = self.normalizer:forward(input) + return self.output +end + +function SpatialContrastiveNormalization:updateGradInput(input, gradOutput) + self.gradInput = self.normalizer:backward(input, gradOutput) + return self.gradInput +end + +function SpatialContrastiveNormalization:type(type) + parent.type(self,type) + self.normalizer:type(type) + return self +end diff --git a/SpatialDivisiveNormalization.lua b/SpatialDivisiveNormalization.lua new file mode 100644 index 0000000..33668e9 --- /dev/null +++ b/SpatialDivisiveNormalization.lua @@ -0,0 +1,133 @@ +local SpatialDivisiveNormalization, parent = torch.class('nn.SpatialDivisiveNormalization','nn.Module') + +function SpatialDivisiveNormalization:__init(nInputPlane, kernel, threshold, thresval) + parent.__init(self) + + -- get args + self.nInputPlane = nInputPlane or 1 + self.kernel = kernel or torch.Tensor(9,9):fill(1) + self.threshold = threshold or 1e-4 + self.thresval = thresval or 1e-4 + local kdim = self.kernel:nDimension() + + -- check args + if kdim ~= 2 and kdim ~= 1 then + error('<SpatialDivisiveNormalization> averaging kernel must be 2D or 1D') + end + if (self.kernel:size(1) % 2) == 0 or (kdim == 2 and (self.kernel:size(2) % 2) == 0) then + error('<SpatialDivisiveNormalization> averaging kernel must have ODD dimensions') + end + + -- padding values + local padH = math.floor(self.kernel:size(1)/2) + local padW = padH + if kdim == 2 then + padW = math.floor(self.kernel:size(2)/2) + end + + -- create convolutional mean estimator + self.meanestimator = nn.Sequential() + self.meanestimator:add(nn.SpatialZeroPadding(padW, padW, padH, padH)) + if kdim == 2 then + self.meanestimator:add(nn.SpatialConvolutionMap(nn.tables.oneToOne(self.nInputPlane), + self.kernel:size(2), self.kernel:size(1))) + else + self.meanestimator:add(nn.SpatialConvolutionMap(nn.tables.oneToOne(self.nInputPlane), + self.kernel:size(1), 1)) + self.meanestimator:add(nn.SpatialConvolutionMap(nn.tables.oneToOne(self.nInputPlane), + 1, self.kernel:size(1))) + end + self.meanestimator:add(nn.Sum(1)) + self.meanestimator:add(nn.Replicate(self.nInputPlane)) + + -- create convolutional std estimator + self.stdestimator = nn.Sequential() + self.stdestimator:add(nn.Square()) + self.stdestimator:add(nn.SpatialZeroPadding(padW, padW, padH, padH)) + if kdim == 2 then + self.stdestimator:add(nn.SpatialConvolutionMap(nn.tables.oneToOne(self.nInputPlane), + self.kernel:size(2), self.kernel:size(1))) + else + self.stdestimator:add(nn.SpatialConvolutionMap(nn.tables.oneToOne(self.nInputPlane), + self.kernel:size(1), 1)) + self.stdestimator:add(nn.SpatialConvolutionMap(nn.tables.oneToOne(self.nInputPlane), + 1, self.kernel:size(1))) + end + self.stdestimator:add(nn.Sum(1)) + self.stdestimator:add(nn.Replicate(self.nInputPlane)) + self.stdestimator:add(nn.Sqrt()) + + -- set kernel and bias + if kdim == 2 then + self.kernel:div(self.kernel:sum() * self.nInputPlane) + for i = 1,self.nInputPlane do + self.meanestimator.modules[2].weight[i] = self.kernel + self.stdestimator.modules[3].weight[i] = self.kernel + end + self.meanestimator.modules[2].bias:zero() + self.stdestimator.modules[3].bias:zero() + else + self.kernel:div(self.kernel:sum() * math.sqrt(self.nInputPlane)) + for i = 1,self.nInputPlane do + self.meanestimator.modules[2].weight[i]:copy(self.kernel) + self.meanestimator.modules[3].weight[i]:copy(self.kernel) + self.stdestimator.modules[3].weight[i]:copy(self.kernel) + self.stdestimator.modules[4].weight[i]:copy(self.kernel) + end + self.meanestimator.modules[2].bias:zero() + self.meanestimator.modules[3].bias:zero() + self.stdestimator.modules[3].bias:zero() + self.stdestimator.modules[4].bias:zero() + end + + -- other operation + self.normalizer = nn.CDivTable() + self.divider = nn.CDivTable() + self.thresholder = nn.Threshold(self.threshold, self.thresval) + + -- coefficient array, to adjust side effects + self.coef = torch.Tensor(1,1,1) +end + +function SpatialDivisiveNormalization:updateOutput(input) + -- compute side coefficients + if (input:size(3) ~= self.coef:size(3)) or (input:size(2) ~= self.coef:size(2)) then + local ones = input.new():resizeAs(input):fill(1) + self.coef = self.meanestimator:updateOutput(ones) + self.coef = self.coef:clone() + end + + -- normalize std dev + self.localstds = self.stdestimator:updateOutput(input) + self.adjustedstds = self.divider:updateOutput{self.localstds, self.coef} + self.thresholdedstds = self.thresholder:updateOutput(self.adjustedstds) + self.output = self.normalizer:updateOutput{input, self.thresholdedstds} + + -- done + return self.output +end + +function SpatialDivisiveNormalization:updateGradInput(input, gradOutput) + -- resize grad + self.gradInput:resizeAs(input):zero() + + -- backprop through all modules + local gradnorm = self.normalizer:updateGradInput({input, self.thresholdedstds}, gradOutput) + local gradadj = self.thresholder:updateGradInput(self.adjustedstds, gradnorm[2]) + local graddiv = self.divider:updateGradInput({self.localstds, self.coef}, gradadj) + self.gradInput:add(self.stdestimator:updateGradInput(input, graddiv[1])) + self.gradInput:add(gradnorm[1]) + + -- done + return self.gradInput +end + +function SpatialDivisiveNormalization:type(type) + parent.type(self,type) + self.meanestimator:type(type) + self.stdestimator:type(type) + self.divider:type(type) + self.normalizer:type(type) + self.thresholder:type(type) + return self +end diff --git a/dok/index.dok b/dok/index.dok index 5a112e5..a687db3 100644 --- a/dok/index.dok +++ b/dok/index.dok @@ -61,7 +61,7 @@ Some important remarks: Building a simple neural network can be achieved by constructing an available layer. A linear neural network (perceptron!) is built only in one line: <file lua> -nn = nn.Linear(10,1) -- perceptron with 10 inputs +mlp = nn.Linear(10,1) -- perceptron with 10 inputs </file> More complex neural networks are easily built using container classes @@ -35,6 +35,7 @@ torch.include('nn', 'CosineDistance.lua') torch.include('nn', 'DotProduct.lua') torch.include('nn', 'Exp.lua') +torch.include('nn', 'Log.lua') torch.include('nn', 'HardTanh.lua') torch.include('nn', 'LogSigmoid.lua') torch.include('nn', 'LogSoftMax.lua') @@ -61,6 +62,8 @@ torch.include('nn', 'SpatialLPPooling.lua') torch.include('nn', 'TemporalConvolution.lua') torch.include('nn', 'TemporalSubSampling.lua') torch.include('nn', 'SpatialSubtractiveNormalization.lua') +torch.include('nn', 'SpatialDivisiveNormalization.lua') +torch.include('nn', 'SpatialContrastiveNormalization.lua') torch.include('nn', 'SpatialZeroPadding.lua') torch.include('nn', 'VolumetricConvolution.lua') @@ -77,6 +80,7 @@ torch.include('nn', 'MSECriterion.lua') torch.include('nn', 'MarginCriterion.lua') torch.include('nn', 'AbsCriterion.lua') torch.include('nn', 'ClassNLLCriterion.lua') +torch.include('nn', 'DistKLDivCriterion.lua') torch.include('nn', 'MultiCriterion.lua') torch.include('nn', 'L1HingeEmbeddingCriterion.lua') torch.include('nn', 'HingeEmbeddingCriterion.lua') diff --git a/test/test.lua b/test/test.lua index 112f041..b8536ca 100644 --- a/test/test.lua +++ b/test/test.lua @@ -75,6 +75,21 @@ function nntest.Exp() mytester:asserteq(berr, 0, torch.typename(module) .. ' - i/o backward err ') end +function nntest.Log() + local ini = math.random(10,20) + local inj = math.random(10,20) + local ink = math.random(10,20) + local input = torch.Tensor(ini,inj,ink):zero() + local module = nn.Log() + + local err = jac.testJacobian(module,input) + mytester:assertlt(err,precision, 'error on state ') + + local ferr,berr = jac.testIO(module,input) + mytester:asserteq(ferr, 0, torch.typename(module) .. ' - i/o forward err ') + mytester:asserteq(berr, 0, torch.typename(module) .. ' - i/o backward err ') +end + function nntest.HardTanh() local ini = math.random(5,10) local inj = math.random(5,10) @@ -542,6 +557,38 @@ function nntest.SpatialSubtractiveNormalization_1dkernel() mytester:asserteq(berr, 0, torch.typename(module) .. ' - i/o backward err ') end +function nntest.SpatialDivisiveNormalization_2dkernel() + local inputSize = math.random(11,20) + local kersize = 9 + local nbfeatures = math.random(5,10) + local kernel = torch.Tensor(kersize,kersize):fill(1) + local module = nn.SpatialDivisiveNormalization(nbfeatures,kernel) + local input = torch.rand(nbfeatures,inputSize,inputSize) + + local err = jac.testJacobian(module,input) + mytester:assertlt(err,precision, 'error on state ') + + local ferr,berr = jac.testIO(module,input) + mytester:asserteq(ferr, 0, torch.typename(module) .. ' - i/o forward err ') + mytester:asserteq(berr, 0, torch.typename(module) .. ' - i/o backward err ') +end + +function nntest.SpatialDivisiveNormalization_1dkernel() + local inputSize = math.random(11,20) + local kersize = 9 + local nbfeatures = math.random(5,10) + local kernel = torch.Tensor(kersize):fill(1) + local module = nn.SpatialDivisiveNormalization(nbfeatures,kernel) + local input = torch.rand(nbfeatures,inputSize,inputSize) + + local err = jac.testJacobian(module,input) + mytester:assertlt(err,precision, 'error on state ') + + local ferr,berr = jac.testIO(module,input) + mytester:asserteq(ferr, 0, torch.typename(module) .. ' - i/o forward err ') + mytester:asserteq(berr, 0, torch.typename(module) .. ' - i/o backward err ') +end + function nntest.SpatialConvolution() local from = math.random(1,10) local to = math.random(1,10) |