adding non-batch mode

8 files changed, 358 insertions, 48 deletions

diff --git a/ReLU.lua b/ReLU.lua
index 7ffcea5..b7f6ae1 100644
--- a/ReLU.lua
+++ b/ReLU.lua
@@ -5,11 +5,17 @@ local errcheck = cudnn.errcheck
 
 function ReLU:__init()
    parent.__init(self)
-   self.iSize = torch.LongStorage(4):fill(0)   
+   self.iSize = torch.LongStorage(4):fill(0)
 end
 
 function ReLU:createIODescriptors(input)
-   if not self.iDesc or not self.oDesc or 
+   local batch = true
+   if input:dim() == 3 then
+      input = input:view(1, input:size(1), input:size(2), input:size(3))
+      batch = false
+   end
+   assert(input:dim() == 4 and input:isContiguous());
+   if not self.iDesc or not self.oDesc or
       input:size(1) ~= self.iSize[1] or input:size(2) ~= self.iSize[2]
    or input:size(3) ~= self.iSize[3] or input:size(4) ~= self.iSize[4] then
       self.iSize = input:size()
@@ -17,26 +23,28 @@ function ReLU:createIODescriptors(input)
       self.output:resizeAs(input)
       self.iDesc = cudnn.toDescriptor(input)
       self.oDesc = cudnn.toDescriptor(self.output)
+      if not batch then
+         self.gradInput = self.gradInput:view(self.gradInput:size(2), self.gradInput:size(3), self.gradInput:size(4))
+         self.output = self.output:view(self.output:size(2), self.output:size(3), self.output:size(4))
+      end
    end
 end
 
 function ReLU:updateOutput(input)
-   assert(input:dim() == 4 and input:isContiguous());
    self:createIODescriptors(input)
    errcheck('cudnnActivationForward', cudnn.handle[cutorch.getDevice()-1], 'CUDNN_ACTIVATION_RELU',
-            self.iDesc[0], input:data(), 
+            self.iDesc[0], input:data(),
             self.oDesc[0], self.output:data());
    return self.output
 end
 
 function ReLU:updateGradInput(input, gradOutput)
-   assert(input:dim() == 4 and input:isContiguous());
-   assert(gradOutput:dim() == 4 and gradOutput:isContiguous());
+   assert((gradOutput:dim() == 4 or gradOutput:dim() == 3) and gradOutput:isContiguous());
    self:createIODescriptors(input)
    errcheck('cudnnActivationBackward', cudnn.handle[cutorch.getDevice()-1], 'CUDNN_ACTIVATION_RELU',
             self.oDesc[0], self.output:data(),
             self.oDesc[0], gradOutput:data(),
-            self.iDesc[0], input:data(), 
+            self.iDesc[0], input:data(),
             self.iDesc[0], self.gradInput:data());
    return self.gradInput
 end
diff --git a/Sigmoid.lua b/Sigmoid.lua
index 9b0dc14..3f1bc8a 100644
--- a/Sigmoid.lua
+++ b/Sigmoid.lua
@@ -5,11 +5,17 @@ local errcheck = cudnn.errcheck
 
 function Sigmoid:__init()
    parent.__init(self)
-   self.iSize = torch.LongStorage(4):fill(0)   
+   self.iSize = torch.LongStorage(4):fill(0)
 end
 
 function Sigmoid:createIODescriptors(input)
-   if not self.iDesc or not self.oDesc or 
+   local batch = true
+   if input:dim() == 3 then
+      input = input:view(1, input:size(1), input:size(2), input:size(3))
+      batch = false
+   end
+   assert(input:dim() == 4 and input:isContiguous());
+   if not self.iDesc or not self.oDesc or
       input:size(1) ~= self.iSize[1] or input:size(2) ~= self.iSize[2]
    or input:size(3) ~= self.iSize[3] or input:size(4) ~= self.iSize[4] then
       self.iSize = input:size()
@@ -17,26 +23,28 @@ function Sigmoid:createIODescriptors(input)
       self.output:resizeAs(input)
       self.iDesc = cudnn.toDescriptor(input)
       self.oDesc = cudnn.toDescriptor(self.output)
+      if not batch then
+         self.gradInput = self.gradInput:view(self.gradInput:size(2), self.gradInput:size(3), self.gradInput:size(4))
+         self.output = self.output:view(self.output:size(2), self.output:size(3), self.output:size(4))
+      end
    end
 end
 
 function Sigmoid:updateOutput(input)
-   assert(input:dim() == 4 and input:isContiguous());
    self:createIODescriptors(input)
    errcheck('cudnnActivationForward', cudnn.handle[cutorch.getDevice()-1], 'CUDNN_ACTIVATION_SIGMOID',
-            self.iDesc[0], input:data(), 
+            self.iDesc[0], input:data(),
             self.oDesc[0], self.output:data());
    return self.output
 end
 
 function Sigmoid:updateGradInput(input, gradOutput)
-   assert(input:dim() == 4 and input:isContiguous());
-   assert(gradOutput:dim() == 4 and gradOutput:isContiguous());
+   assert((gradOutput:dim() == 4 or gradOutput:dim() == 3) and gradOutput:isContiguous());
    self:createIODescriptors(input)
    errcheck('cudnnActivationBackward', cudnn.handle[cutorch.getDevice()-1], 'CUDNN_ACTIVATION_SIGMOID',
             self.oDesc[0], self.output:data(),
             self.oDesc[0], gradOutput:data(),
-            self.iDesc[0], input:data(), 
+            self.iDesc[0], input:data(),
             self.iDesc[0], self.gradInput:data());
    return self.gradInput
 end
diff --git a/SpatialConvolution.lua b/SpatialConvolution.lua
index 76bb33e..d175f90 100644
--- a/SpatialConvolution.lua
+++ b/SpatialConvolution.lua
@@ -30,6 +30,12 @@ function SpatialConvolution:resetWeightDescriptors()
 end
 
 function SpatialConvolution:createIODescriptors(input)
+   local batch = true
+   if input:dim() == 3 then
+      input = input:view(1, input:size(1), input:size(2), input:size(3))
+      batch = false
+   end
+   assert(input:dim() == 4 and input:isContiguous());
    if not self.iDesc or not self.oDesc or
       input:size(1) ~= self.iSize[1] or input:size(2) ~= self.iSize[2]
    or input:size(3) ~= self.iSize[3] or input:size(4) ~= self.iSize[4] then
@@ -57,11 +63,14 @@ function SpatialConvolution:createIODescriptors(input)
          self.output:resize(oSize:long():storage())
          -- create descriptor for output
          self.oDesc = cudnn.toDescriptor(self.output)
+         if not batch then
+            self.gradInput = self.gradInput:view(self.gradInput:size(2), self.gradInput:size(3), self.gradInput:size(4))
+            self.output = self.output:view(self.output:size(2), self.output:size(3), self.output:size(4))
+         end
    end
 end
 
 function SpatialConvolution:updateOutput(input)
-   assert(input:dim() == 4 and input:isContiguous());
    if not self.weightDesc then self:resetWeightDescriptors() end
    self:createIODescriptors(input)
    errcheck('cudnnConvolutionForward', cudnn.handle[cutorch.getDevice()-1],
@@ -78,8 +87,7 @@ end
 
 function SpatialConvolution:updateGradInput(input, gradOutput)
    if not self.gradInput then return end
-   assert(input:dim() == 4 and input:isContiguous());
-   assert(gradOutput:dim() == 4 and gradOutput:isContiguous());
+   assert((gradOutput:dim() == 3 or gradOutput:dim() == 4) and gradOutput:isContiguous());
    if not self.weightDesc then self:resetWeightDescriptors() end
    self:createIODescriptors(input)
    errcheck('cudnnConvolutionBackwardData', cudnn.handle[cutorch.getDevice()-1],
@@ -93,8 +101,7 @@ end
 
 function SpatialConvolution:accGradParameters(input, gradOutput, scale)
    assert(scale == nil or scale == 1)
-   assert(input:dim() == 4 and input:isContiguous());
-   assert(gradOutput:dim() == 4 and gradOutput:isContiguous());
+   assert((gradOutput:dim() == 3 or gradOutput:dim() == 4) and gradOutput:isContiguous());
    self:createIODescriptors(input)
    if not self.weightDesc then self:resetWeightDescriptors() end
    -- gradBias
diff --git a/SpatialMaxPooling.lua b/SpatialMaxPooling.lua
index 53c93a9..8bd1bf6 100644
--- a/SpatialMaxPooling.lua
+++ b/SpatialMaxPooling.lua
@@ -19,14 +19,20 @@ function SpatialMaxPooling:resetPoolDescriptors()
    errcheck('cudnnCreatePoolingDescriptor', self.poolDesc)
    errcheck('cudnnSetPoolingDescriptor', self.poolDesc[0], self.mode,
             self.kH, self.kW, self.dH, self.dW);
-   local function destroyPoolDesc(d) 
+   local function destroyPoolDesc(d)
       errcheck('cudnnDestroyPoolingDescriptor', d[0]);
    end
    ffi.gc(self.poolDesc, destroyPoolDesc)
 end
 
 function SpatialMaxPooling:createIODescriptors(input)
-   if not self.iDesc or not self.oDesc or 
+   local batch = true
+   if input:dim() == 3 then
+      input = input:view(1, input:size(1), input:size(2), input:size(3))
+      batch = false
+   end
+   assert(input:dim() == 4 and input:isContiguous());
+   if not self.iDesc or not self.oDesc or
       input:size(1) ~= self.iSize[1] or input:size(2) ~= self.iSize[2]
    or input:size(3) ~= self.iSize[3] or input:size(4) ~= self.iSize[4] then
       self.iSize = input:size()
@@ -40,22 +46,24 @@ function SpatialMaxPooling:createIODescriptors(input)
       -- create input/output descriptor
       self.iDesc = cudnn.toDescriptor(input)
       self.oDesc = cudnn.toDescriptor(self.output)
+      if not batch then
+         self.gradInput = self.gradInput:view(self.gradInput:size(2), self.gradInput:size(3), self.gradInput:size(4))
+         self.output = self.output:view(self.output:size(2), self.output:size(3), self.output:size(4))
+      end
    end
 end
 
 function SpatialMaxPooling:updateOutput(input)
-   assert(input:dim() == 4 and input:isContiguous());
    if not self.poolDesc then self:resetPoolDescriptors() end
    self:createIODescriptors(input)
    errcheck('cudnnPoolingForward', cudnn.handle[cutorch.getDevice()-1], self.poolDesc[0],
-            self.iDesc[0], input:data(), 
+            self.iDesc[0], input:data(),
             self.oDesc[0], self.output:data());
    return self.output
 end
 
 function SpatialMaxPooling:updateGradInput(input, gradOutput)
-   assert(input:dim() == 4 and input:isContiguous());
-   assert(gradOutput:dim() == 4);
+   assert(gradOutput:dim() == 3 or gradOutput:dim() == 4);
    if not gradOutput:isContiguous() then
       self._gradOutput = self._gradOutput or gradOutput.new()
       self._gradOutput:resizeAs(gradOutput):copy(gradOutput)
@@ -66,8 +74,7 @@ function SpatialMaxPooling:updateGradInput(input, gradOutput)
    errcheck('cudnnPoolingBackward', cudnn.handle[cutorch.getDevice()-1], self.poolDesc[0],
             self.oDesc[0], self.output:data(),
             self.oDesc[0], gradOutput:data(),
-            self.iDesc[0], input:data(), 
+            self.iDesc[0], input:data(),
             self.iDesc[0], self.gradInput:data());
    return self.gradInput
 end
-
diff --git a/SpatialSoftMax.lua b/SpatialSoftMax.lua
index ee93749..e245f24 100644
--- a/SpatialSoftMax.lua
+++ b/SpatialSoftMax.lua
@@ -15,6 +15,12 @@ function SpatialSoftMax:__init(fast)
 end
 
 function SpatialSoftMax:createIODescriptors(input)
+   local batch = true
+   if input:dim() == 3 then
+      input = input:view(1, input:size(1), input:size(2), input:size(3))
+      batch = false
+   end
+   assert(input:dim() == 4 and input:isContiguous());
    if not self.iDesc or not self.oDesc or
       input:size(1) ~= self.iSize[1] or input:size(2) ~= self.iSize[2]
    or input:size(3) ~= self.iSize[3] or input:size(4) ~= self.iSize[4] then
@@ -23,11 +29,14 @@ function SpatialSoftMax:createIODescriptors(input)
       self.output:resizeAs(input)
       self.iDesc = cudnn.toDescriptor(input)
       self.oDesc = cudnn.toDescriptor(self.output)
+      if not batch then
+         self.gradInput = self.gradInput:view(self.gradInput:size(2), self.gradInput:size(3), self.gradInput:size(4))
+         self.output = self.output:view(self.output:size(2), self.output:size(3), self.output:size(4))
+      end
    end
 end
 
 function SpatialSoftMax:updateOutput(input)
-   assert(input:dim() == 4 and input:isContiguous());
    self:createIODescriptors(input)
    errcheck('cudnnSoftmaxForward',
             cudnn.handle[cutorch.getDevice()-1],
@@ -38,8 +47,7 @@ function SpatialSoftMax:updateOutput(input)
 end
 
 function SpatialSoftMax:updateGradInput(input, gradOutput)
-   assert(input:dim() == 4 and input:isContiguous());
-   assert(gradOutput:dim() == 4 and gradOutput:isContiguous());
+   assert((gradOutput:dim() == 4 or gradOutput:dim() == 3) and gradOutput:isContiguous());
    self:createIODescriptors(input)
    errcheck('cudnnSoftmaxBackward',
             cudnn.handle[cutorch.getDevice()-1],
diff --git a/Tanh.lua b/Tanh.lua
index ee1e264..46fbe35 100644
--- a/Tanh.lua
+++ b/Tanh.lua
@@ -5,11 +5,17 @@ local errcheck = cudnn.errcheck
 
 function Tanh:__init()
    parent.__init(self)
-   self.iSize = torch.LongStorage(4):fill(0)   
+   self.iSize = torch.LongStorage(4):fill(0)
 end
 
 function Tanh:createIODescriptors(input)
-   if not self.iDesc or not self.oDesc or 
+   local batch = true
+   if input:dim() == 3 then
+      input = input:view(1, input:size(1), input:size(2), input:size(3))
+      batch = false
+   end
+   assert(input:dim() == 4 and input:isContiguous());
+   if not self.iDesc or not self.oDesc or
       input:size(1) ~= self.iSize[1] or input:size(2) ~= self.iSize[2]
    or input:size(3) ~= self.iSize[3] or input:size(4) ~= self.iSize[4] then
       self.iSize = input:size()
@@ -17,26 +23,28 @@ function Tanh:createIODescriptors(input)
       self.output:resizeAs(input)
       self.iDesc = cudnn.toDescriptor(input)
       self.oDesc = cudnn.toDescriptor(self.output)
+      if not batch then
+         self.gradInput = self.gradInput:view(self.gradInput:size(2), self.gradInput:size(3), self.gradInput:size(4))
+         self.output = self.output:view(self.output:size(2), self.output:size(3), self.output:size(4))
+      end
    end
 end
 
 function Tanh:updateOutput(input)
-   assert(input:dim() == 4 and input:isContiguous());
    self:createIODescriptors(input)
    errcheck('cudnnActivationForward', cudnn.handle[cutorch.getDevice()-1], 'CUDNN_ACTIVATION_TANH',
-            self.iDesc[0], input:data(), 
+            self.iDesc[0], input:data(),
             self.oDesc[0], self.output:data());
    return self.output
 end
 
 function Tanh:updateGradInput(input, gradOutput)
-   assert(input:dim() == 4 and input:isContiguous());
-   assert(gradOutput:dim() == 4 and gradOutput:isContiguous());
+   assert((gradOutput:dim() == 4 or gradOutput:dim() == 3) and gradOutput:isContiguous());
    self:createIODescriptors(input)
    errcheck('cudnnActivationBackward', cudnn.handle[cutorch.getDevice()-1], 'CUDNN_ACTIVATION_TANH',
             self.oDesc[0], self.output:data(),
             self.oDesc[0], gradOutput:data(),
-            self.iDesc[0], input:data(), 
+            self.iDesc[0], input:data(),
             self.iDesc[0], self.gradInput:data());
    return self.gradInput
 end
diff --git a/init.lua b/init.lua
index c27aec5..66fb73d 100644
--- a/init.lua
+++ b/init.lua
@@ -27,25 +27,26 @@ local function destroy(handle)
    local currentDevice = cutorch.getDevice()
    for i=1,numDevices do
       cutorch.setDevice(i)
-      errcheck('cudnnDestroy', handle[i-1]); 
+      errcheck('cudnnDestroy', handle[i-1]);
    end
    cutorch.setDevice(currentDevice)
 end
 ffi.gc(cudnn.handle, destroy)
 
 function cudnn.toDescriptor(t)
+   if t:dim() == 3 then t = t:view(1, t:size(1), t:size(2), t:size(3)) end
    assert(t:dim() == 4);
    assert(torch.typename(t) == 'torch.CudaTensor')
    local descriptor = ffi.new('struct cudnnTensor4dStruct*[1]')
    -- create descriptor
    errcheck('cudnnCreateTensor4dDescriptor', descriptor)
    -- set gc hook
-   local function destroy(d) 
-      errcheck('cudnnDestroyTensor4dDescriptor', d[0]); 
+   local function destroy(d)
+      errcheck('cudnnDestroyTensor4dDescriptor', d[0]);
    end
    ffi.gc(descriptor, destroy)
    -- set descriptor
-   errcheck('cudnnSetTensor4dDescriptorEx', descriptor[0], 'CUDNN_DATA_FLOAT', 
+   errcheck('cudnnSetTensor4dDescriptorEx', descriptor[0], 'CUDNN_DATA_FLOAT',
             t:size(1), t:size(2), t:size(3), t:size(4),
             t:stride(1), t:stride(2), t:stride(3), t:stride(4))
    return descriptor
diff --git a/test/test.lua b/test/test.lua
index 6965219..3750418 100644
--- a/test/test.lua
+++ b/test/test.lua
@@ -9,7 +9,7 @@ local nloop = 1
 local times = {}
 
 
-function cudnntest.SpatialConvolution_forward()
+function cudnntest.SpatialConvolution_forward_batch()
    local bs = math.random(1,32)
    local from = math.random(1,32)
    local to = math.random(1,64)
@@ -36,7 +36,7 @@ function cudnntest.SpatialConvolution_forward()
 end
 
 
-function cudnntest.SpatialConvolution_backward()
+function cudnntest.SpatialConvolution_backward_batch()
    local bs = math.random(1,32)
    local from = math.random(1,32)
    local to = math.random(1,64)
@@ -84,7 +84,83 @@ function cudnntest.SpatialConvolution_backward()
    mytester:assertlt(berror:abs():max(), precision_backward, 'error on bias (backward) ')
 end
 
-function cudnntest.SpatialMaxPooling()
+function cudnntest.SpatialConvolution_forward_single()
+   local from = math.random(1,32)
+   local to = math.random(1,64)
+   local ki = math.random(3,15)
+   local kj = math.random(3,15)
+   local si = 1 -- not supported by CPU version yet
+   local sj = si
+   local outi = math.random(1,64)
+   local outj = math.random(1,64)
+   local ini = (outi-1)*si+ki
+   local inj = (outj-1)*sj+kj
+
+   local input = torch.randn(from,inj,ini):cuda()
+   local sconv = nn.SpatialConvolutionMM(from,to,ki,kj,si,sj):cuda()
+   local groundtruth = sconv:forward(input)
+   cutorch.synchronize()
+   local gconv = cudnn.SpatialConvolution(from,to,ki,kj,si,sj):cuda()
+   gconv.weight:copy(sconv.weight)
+   gconv.bias:copy(sconv.bias)
+   local rescuda = gconv:forward(input)
+   cutorch.synchronize()
+   mytester:asserteq(rescuda:dim(), 3, 'error in dimension')
+   local error = rescuda:float() - groundtruth:float()
+   mytester:assertlt(error:abs():max(), precision_forward, 'error on state (forward) ')
+end
+
+
+function cudnntest.SpatialConvolution_backward_single()
+   local from = math.random(1,32)
+   local to = math.random(1,64)
+   local ki = math.random(3,15)
+   local kj = math.random(3,15)
+   local si = 1 -- not supported by CPU version yet
+   local sj = si
+   local outi = math.random(1,64)
+   local outj = math.random(1,64)
+   local ini = (outi-1)*si+ki
+   local inj = (outj-1)*sj+kj
+
+   local input = torch.randn(from,inj,ini):cuda()
+   local gradOutput = torch.randn(to,outj,outi):cuda()
+   local sconv = nn.SpatialConvolutionMM(from,to,ki,kj,si,sj):cuda()
+   sconv:forward(input)
+   sconv:zeroGradParameters()
+   local groundgrad = sconv:backward(input, gradOutput)
+   cutorch.synchronize()
+   local groundweight = sconv.gradWeight
+   local groundbias = sconv.gradBias
+
+   local gconv = cudnn.SpatialConvolution(from,to,ki,kj,si,sj):cuda()
+   gconv.weight:copy(sconv.weight)
+   gconv.bias:copy(sconv.bias)
+   gconv:forward(input)
+
+   -- serialize and deserialize
+   torch.save('modelTemp.t7', gconv)
+   gconv = torch.load('modelTemp.t7')
+
+   gconv:forward(input)
+   gconv:zeroGradParameters()
+   local rescuda = gconv:backward(input, gradOutput)
+   cutorch.synchronize()
+   mytester:asserteq(rescuda:dim(), 3, 'error in dimension')
+   local weightcuda = gconv.gradWeight
+   local biascuda = gconv.gradBias
+
+   local error = rescuda:float() - groundgrad:float()
+   local werror = weightcuda:float() - groundweight:float()
+   local berror = biascuda:float() - groundbias:float()
+
+   mytester:assertlt(error:abs():max(), precision_backward, 'error on state (backward) ')
+   mytester:assertlt(werror:abs():max(), precision_backward, 'error on weight (backward) ')
+   mytester:assertlt(berror:abs():max(), precision_backward, 'error on bias (backward) ')
+end
+
+
+function cudnntest.SpatialMaxPooling_batch()
    local bs = math.random(1,32)
    local from = math.random(1,32)
    local ki = math.random(2,4)
@@ -110,13 +186,83 @@ function cudnntest.SpatialMaxPooling()
    local rescuda = gconv:forward(input)
    local resgrad = gconv:backward(input, gradOutput)
    cutorch.synchronize()
+   mytester:asserteq(rescuda:dim(), 4, 'error in dimension')
+   mytester:asserteq(resgrad:dim(), 4, 'error in dimension')
+   local error = rescuda:float() - groundtruth:float()
+   mytester:assertlt(error:abs():max(), precision_forward, 'error on state (forward) ')
+   error = resgrad:float() - groundgrad:float()
+   mytester:assertlt(error:abs():max(), precision_backward, 'error on state (backward) ')
+end
+
+function cudnntest.SpatialMaxPooling_single()
+   local from = math.random(1,32)
+   local ki = math.random(2,4)
+   local kj = math.random(2,4)
+   local si = ki
+   local sj = kj
+   local outi = math.random(1,64)
+   local outj = math.random(1,64)
+   local ini = (outi-1)*si+ki
+   local inj = (outj-1)*sj+kj
+   local input = torch.randn(from,inj,ini):cuda()
+   local gradOutput = torch.randn(from,outj,outi):cuda()
+
+   local sconv = nn.SpatialMaxPooling(ki,kj,si,sj):cuda()
+   local groundtruth = sconv:forward(input)
+   local groundgrad = sconv:backward(input, gradOutput)
+   cutorch.synchronize()
+   local gconv = cudnn.SpatialMaxPooling(ki,kj,si,sj):cuda()
+   local rescuda = gconv:forward(input)
+   -- serialize and deserialize
+   torch.save('modelTemp.t7', gconv)
+   gconv = torch.load('modelTemp.t7')
+   local rescuda = gconv:forward(input)
+   local resgrad = gconv:backward(input, gradOutput)
+   cutorch.synchronize()
+   mytester:asserteq(rescuda:dim(), 3, 'error in dimension')
+   mytester:asserteq(resgrad:dim(), 3, 'error in dimension')
+   local error = rescuda:float() - groundtruth:float()
+   mytester:assertlt(error:abs():max(), precision_forward, 'error on state (forward) ')
+   error = resgrad:float() - groundgrad:float()
+   mytester:assertlt(error:abs():max(), precision_backward, 'error on state (backward) ')
+end
+
+function cudnntest.ReLU_single()
+   local from = math.random(1,32)
+   local ki = math.random(2,4)
+   local kj = math.random(2,4)
+   local si = ki
+   local sj = kj
+   local outi = math.random(1,64)
+   local outj = math.random(1,64)
+   local ini = outi
+   local inj = outj
+   local input = torch.randn(from,inj,ini):cuda()
+   local gradOutput = torch.randn(from,outj,outi):cuda()
+
+   local sconv = nn.ReLU():cuda()
+   local groundtruth = sconv:forward(input)
+   local groundgrad = sconv:backward(input, gradOutput)
+   cutorch.synchronize()
+   local gconv = cudnn.ReLU():cuda()
+   local rescuda = gconv:forward(input)
+
+   -- serialize and deserialize
+   torch.save('modelTemp.t7', gconv)
+   gconv = torch.load('modelTemp.t7')
+
+   local rescuda = gconv:forward(input)
+   local resgrad = gconv:backward(input, gradOutput)
+   cutorch.synchronize()
+   mytester:asserteq(rescuda:dim(), 3, 'error in dimension')
+   mytester:asserteq(resgrad:dim(), 3, 'error in dimension')
    local error = rescuda:float() - groundtruth:float()
    mytester:assertlt(error:abs():max(), precision_forward, 'error on state (forward) ')
    error = resgrad:float() - groundgrad:float()
    mytester:assertlt(error:abs():max(), precision_backward, 'error on state (backward) ')
 end
 
-function cudnntest.ReLU()
+function cudnntest.ReLU_batch()
    local bs = math.random(1,32)
    local from = math.random(1,32)
    local ki = math.random(2,4)
@@ -144,13 +290,50 @@ function cudnntest.ReLU()
    local rescuda = gconv:forward(input)
    local resgrad = gconv:backward(input, gradOutput)
    cutorch.synchronize()
+   mytester:asserteq(rescuda:dim(), 4, 'error in dimension')
+   mytester:asserteq(resgrad:dim(), 4, 'error in dimension')
    local error = rescuda:float() - groundtruth:float()
    mytester:assertlt(error:abs():max(), precision_forward, 'error on state (forward) ')
    error = resgrad:float() - groundgrad:float()
    mytester:assertlt(error:abs():max(), precision_backward, 'error on state (backward) ')
 end
 
-function cudnntest.Tanh()
+function cudnntest.Tanh_single()
+   local from = math.random(1,32)
+   local ki = math.random(2,4)
+   local kj = math.random(2,4)
+   local si = ki
+   local sj = kj
+   local outi = math.random(1,64)
+   local outj = math.random(1,64)
+   local ini = outi
+   local inj = outj
+   local input = torch.randn(from,inj,ini):cuda()
+   local gradOutput = torch.randn(from,outj,outi):cuda()
+
+   local sconv = nn.Tanh():cuda()
+   local groundtruth = sconv:forward(input)
+   local groundgrad = sconv:backward(input, gradOutput)
+   cutorch.synchronize()
+   local gconv = cudnn.Tanh():cuda()
+   local rescuda = gconv:forward(input)
+
+   -- serialize and deserialize
+   torch.save('modelTemp.t7', gconv)
+   gconv = torch.load('modelTemp.t7')
+
+   local rescuda = gconv:forward(input)
+   local resgrad = gconv:backward(input, gradOutput)
+   cutorch.synchronize()
+   mytester:asserteq(rescuda:dim(), 3, 'error in dimension')
+   mytester:asserteq(resgrad:dim(), 3, 'error in dimension')
+   local error = rescuda:float() - groundtruth:float()
+   mytester:assertlt(error:abs():max(), precision_forward, 'error on state (forward) ')
+   error = resgrad:float() - groundgrad:float()
+   mytester:assertlt(error:abs():max(), precision_backward, 'error on state (backward) ')
+end
+
+function cudnntest.Tanh_batch()
    local bs = math.random(1,32)
    local from = math.random(1,32)
    local ki = math.random(2,4)
@@ -178,13 +361,50 @@ function cudnntest.Tanh()
    local rescuda = gconv:forward(input)
    local resgrad = gconv:backward(input, gradOutput)
    cutorch.synchronize()
+   mytester:asserteq(rescuda:dim(), 4, 'error in dimension')
+   mytester:asserteq(resgrad:dim(), 4, 'error in dimension')
+   local error = rescuda:float() - groundtruth:float()
+   mytester:assertlt(error:abs():max(), precision_forward, 'error on state (forward) ')
+   error = resgrad:float() - groundgrad:float()
+   mytester:assertlt(error:abs():max(), precision_backward, 'error on state (backward) ')
+end
+
+function cudnntest.Sigmoid_single()
+   local from = math.random(1,32)
+   local ki = math.random(2,4)
+   local kj = math.random(2,4)
+   local si = ki
+   local sj = kj
+   local outi = math.random(1,64)
+   local outj = math.random(1,64)
+   local ini = outi
+   local inj = outj
+   local input = torch.randn(from,inj,ini):cuda()
+   local gradOutput = torch.randn(from,outj,outi):cuda()
+
+   local sconv = nn.Sigmoid():cuda()
+   local groundtruth = sconv:forward(input)
+   local groundgrad = sconv:backward(input, gradOutput)
+   cutorch.synchronize()
+   local gconv = cudnn.Sigmoid():cuda()
+   local rescuda = gconv:forward(input)
+
+   -- serialize and deserialize
+   torch.save('modelTemp.t7', gconv)
+   gconv = torch.load('modelTemp.t7')
+
+   local rescuda = gconv:forward(input)
+   local resgrad = gconv:backward(input, gradOutput)
+   cutorch.synchronize()
+   mytester:asserteq(rescuda:dim(), 3, 'error in dimension')
+   mytester:asserteq(resgrad:dim(), 3, 'error in dimension')
    local error = rescuda:float() - groundtruth:float()
    mytester:assertlt(error:abs():max(), precision_forward, 'error on state (forward) ')
    error = resgrad:float() - groundgrad:float()
    mytester:assertlt(error:abs():max(), precision_backward, 'error on state (backward) ')
 end
 
-function cudnntest.Sigmoid()
+function cudnntest.Sigmoid_batch()
    local bs = math.random(1,32)
    local from = math.random(1,32)
    local ki = math.random(2,4)
@@ -212,13 +432,53 @@ function cudnntest.Sigmoid()
    local rescuda = gconv:forward(input)
    local resgrad = gconv:backward(input, gradOutput)
    cutorch.synchronize()
+   mytester:asserteq(rescuda:dim(), 4, 'error in dimension')
+   mytester:asserteq(resgrad:dim(), 4, 'error in dimension')
    local error = rescuda:float() - groundtruth:float()
    mytester:assertlt(error:abs():max(), precision_forward, 'error on state (forward) ')
    error = resgrad:float() - groundgrad:float()
    mytester:assertlt(error:abs():max(), precision_backward, 'error on state (backward) ')
 end
 
-function cudnntest.SoftMax()
+function cudnntest.SoftMax_single()
+   local from = math.random(1,32)
+   local ki = math.random(2,4)
+   local kj = math.random(2,4)
+   local si = ki
+   local sj = kj
+   local outi = math.random(1,64)
+   local outj = math.random(1,64)
+   local ini = outi
+   local inj = outj
+   local input = torch.randn(from,inj,ini):cuda()
+   local gradOutput = torch.randn(from,outj,outi):cuda()
+
+   local sconv = nn.SoftMax():cuda()
+   local groundtruth = sconv:forward(input:view(-1))
+   local groundgrad = sconv:backward(input, gradOutput)
+   cutorch.synchronize()
+   local gconv = cudnn.SoftMax():cuda()
+   local rescuda = gconv:forward(input)
+
+   -- serialize and deserialize
+   torch.save('modelTemp.t7', gconv)
+   gconv = torch.load('modelTemp.t7')
+
+   local rescuda = gconv:forward(input)
+   local resgrad = gconv:backward(input, gradOutput)
+   cutorch.synchronize()
+   mytester:asserteq(rescuda:dim(), 3, 'error in dimension')
+   mytester:asserteq(resgrad:dim(), 3, 'error in dimension')
+
+   local error = rescuda:float() - groundtruth:float()
+   mytester:assertlt(error:abs():max(),
+                     precision_forward, 'error on state (forward) ')
+   error = resgrad:float() - groundgrad:float()
+   mytester:assertlt(error:abs():max(),
+                     precision_backward, 'error on state (backward) ')
+end
+
+function cudnntest.SoftMax_batch()
    local bs = math.random(1,32)
    local from = math.random(1,32)
    local ki = math.random(2,4)
@@ -246,6 +506,9 @@ function cudnntest.SoftMax()
    local rescuda = gconv:forward(input)
    local resgrad = gconv:backward(input, gradOutput)
    cutorch.synchronize()
+   mytester:asserteq(rescuda:dim(), 4, 'error in dimension')
+   mytester:asserteq(resgrad:dim(), 4, 'error in dimension')
+
    local error = rescuda:float() - groundtruth:float()
    mytester:assertlt(error:abs():max(),
                      precision_forward, 'error on state (forward) ')