From 1ee64da4abbafd92c6bea5121e9a11e55ef1cbcb Mon Sep 17 00:00:00 2001 From: koray kavukcuoglu Date: Thu, 3 Jan 2013 14:11:51 +0000 Subject: New NN classes extra/nn/L1Cost.lua : L1 penalty extra/nn/SpatialFullConvolution.lua : full convolution extra/nn/SpatialFullConvolutionMap.lua : full convolution with connection table extra/nn/TanhShrink.lua : shrinkage with x-tanh(x) extra/nn/WeightedMSECriterion.lua : mean squared error with weighting mask on the target Add new nn classes that are used commonly for unsupervised training of convolutional auto encoders --- generic/L1Cost.c | 49 ++++++++ generic/SpatialConvolutionMap.c | 2 +- generic/SpatialFullConvolution.c | 191 ++++++++++++++++++++++++++++++ generic/SpatialFullConvolutionMap.c | 225 ++++++++++++++++++++++++++++++++++++ 4 files changed, 466 insertions(+), 1 deletion(-) create mode 100644 generic/L1Cost.c create mode 100644 generic/SpatialFullConvolution.c create mode 100644 generic/SpatialFullConvolutionMap.c (limited to 'generic') diff --git a/generic/L1Cost.c b/generic/L1Cost.c new file mode 100644 index 0000000..a450e06 --- /dev/null +++ b/generic/L1Cost.c @@ -0,0 +1,49 @@ +#ifndef TH_GENERIC_FILE +#define TH_GENERIC_FILE "generic/L1Cost.c" +#else + +static int nn_(L1Cost_updateOutput)(lua_State *L) +{ + THTensor *input = luaT_checkudata(L, 2, torch_Tensor); + accreal sum; + + sum = 0; + TH_TENSOR_APPLY(real, input, sum += fabs(*input_data);); + + lua_pushnumber(L, sum); + lua_setfield(L, 1, "output"); + + lua_pushnumber(L, sum); + return 1; +} + +static int nn_(L1Cost_updateGradInput)(lua_State *L) +{ + THTensor *input = luaT_checkudata(L, 2, torch_Tensor); + THTensor *gradInput = luaT_getfieldcheckudata(L, 1, "gradInput", torch_Tensor); + + THTensor_(resizeAs)(gradInput, input); + TH_TENSOR_APPLY2(real, gradInput, real, input, + if (*input_data > 0) + *gradInput_data = 1; + else if (*input_data < 0) + *gradInput_data = -1; + else + *gradInput_data = 0;); + return 1; +} + +static const struct luaL_Reg nn_(L1Cost__) [] = { + {"L1Cost_updateOutput", nn_(L1Cost_updateOutput)}, + {"L1Cost_updateGradInput", nn_(L1Cost_updateGradInput)}, + {NULL, NULL} +}; + +static void nn_(L1Cost_init)(lua_State *L) +{ + luaT_pushmetatable(L, torch_Tensor); + luaT_registeratname(L, nn_(L1Cost__), "nn"); + lua_pop(L,1); +} + +#endif diff --git a/generic/SpatialConvolutionMap.c b/generic/SpatialConvolutionMap.c index 4c289fb..a1d20bc 100644 --- a/generic/SpatialConvolutionMap.c +++ b/generic/SpatialConvolutionMap.c @@ -18,7 +18,7 @@ static int nn_(SpatialConvolutionMap_updateOutput)(lua_State *L) THTensor *output = luaT_getfieldcheckudata(L, 1, "output", torch_Tensor); luaL_argcheck(L, input->nDimension == 3, 2, "3D tensor expected"); - luaL_argcheck(L, input->size[0] == nInputPlane, 2, "invalid number of input planes"); + luaL_argcheck(L, input->size[0] >= nInputPlane, 2, "invalid number of input planes"); luaL_argcheck(L, input->size[2] >= kW && input->size[1] >= kH, 2, "input image smaller than kernel size"); THTensor_(resize3d)(output, nOutputPlane, diff --git a/generic/SpatialFullConvolution.c b/generic/SpatialFullConvolution.c new file mode 100644 index 0000000..cb2e340 --- /dev/null +++ b/generic/SpatialFullConvolution.c @@ -0,0 +1,191 @@ +#ifndef TH_GENERIC_FILE +#define TH_GENERIC_FILE "generic/SpatialFullConvolution.c" +#else + +static int nn_(SpatialFullConvolution_updateOutput)(lua_State *L) +{ + THTensor *input = luaT_checkudata(L, 2, torch_Tensor); + int dW = luaT_getfieldcheckint(L, 1, "dW"); + int dH = luaT_getfieldcheckint(L, 1, "dH"); + + THTensor *weight = luaT_getfieldcheckudata(L, 1, "weight", torch_Tensor); + THTensor *bias = luaT_getfieldcheckudata(L, 1, "bias", torch_Tensor); + THTensor *output = luaT_getfieldcheckudata(L, 1, "output", torch_Tensor); + + luaL_argcheck(L, input->nDimension == 3 || input->nDimension == 4, 2, "3D or 4D(batch mode) tensor expected"); + int dimw = 2; + int dimh = 1; + if (input->nDimension == 4) { + dimw++; + dimh++; + } + + long nOutputPlane = weight->size[1]; + long kW = weight->size[3]; + long kH = weight->size[2]; + long inputWidth = input->size[dimw]; + long inputHeight = input->size[dimh]; + long outputWidth = (inputWidth - 1) * dW + kW; + long outputHeight = (inputHeight - 1) * dH + kH; + + if (input->nDimension == 3) + { + THTensor_(resize3d)(output, nOutputPlane, outputHeight, outputWidth); + /* add bias */ + long i; + real* bias_data = THTensor_(data)(bias); + real* output_data = THTensor_(data)(output); +#pragma omp parallel for private(i) + for (i=0; isize[0]; i++) + { + /*THTensor_(select)(outn,output,0,i);*/ + /*TH_TENSOR_APPLY(real,outn, *outn_data = bias_data[i];);*/ + real *ptr_output = output_data + i*outputWidth*outputHeight; + long j; + for(j = 0; j < outputWidth*outputHeight; j++) + ptr_output[j] = bias_data[i]; + } + + /* do convolutions */ + THTensor *tweight = THTensor_(newTranspose)(weight,0,1); + THTensor_(conv2Dmv)(output, 1.0, 1.0, input, tweight, dH, dW, "F", "C"); + THTensor_(free)(tweight); + } + else + { + THTensor_(resize4d)(output, input->size[0], nOutputPlane, outputHeight, outputWidth); + real* bias_data = THTensor_(data)(bias); + real* output_data = THTensor_(data)(output); + + long p; +#pragma omp parallel for private(p) + for (p=0; psize[0]; p++) + { + /* BIAS */ + long i; + for (i=0; isize[0]; i++) + { + real *ptr_output = output_data + p*nOutputPlane*outputWidth*outputHeight + i*outputWidth*outputHeight; + long j; + for(j = 0; j < outputWidth*outputHeight; j++) + ptr_output[j] = bias_data[i]; + } + } + /* do convolutions */ + THTensor *tweight = THTensor_(newTranspose)(weight,0,1); + THTensor_(conv2Dmm)(output, 1.0, 1.0, input, tweight, dH, dW, "F", "C"); + THTensor_(free)(tweight); + } + return 1; +} + + +static int nn_(SpatialFullConvolution_updateGradInput)(lua_State *L) +{ + THTensor *input = luaT_checkudata(L, 2, torch_Tensor); + THTensor *gradOutput = luaT_checkudata(L, 3, torch_Tensor); + int dW = luaT_getfieldcheckint(L, 1, "dW"); + int dH = luaT_getfieldcheckint(L, 1, "dH"); + + THTensor *weight = luaT_getfieldcheckudata(L, 1, "weight", torch_Tensor); + THTensor *gradInput = luaT_getfieldcheckudata(L, 1, "gradInput", torch_Tensor); + + long nOutputPlane = weight->size[1]; + THArgCheck( nOutputPlane == gradOutput->size[input->nDimension == 4 ? 1 : 0], 1, "Number of output features is not equal to nOutputPlane" ); + + if (input->nDimension == 3) + { + /* gradient to input */ + THTensor_(conv2Dmv)(gradInput, 0.0, 1.0, gradOutput, weight, dH, dW, "V", "X"); + } + else + { + /* gradient to input */ + THTensor_(conv2Dmm)(gradInput, 0.0, 1.0, gradOutput, weight, dH, dW, "V", "X"); + } + + return 1; +} + +static int nn_(SpatialFullConvolution_accGradParameters)(lua_State *L) +{ + THTensor *input = luaT_checkudata(L, 2, torch_Tensor); + THTensor *gradOutput = luaT_checkudata(L, 3, torch_Tensor); + real scale = luaL_optnumber(L, 4, 1); + int dW = luaT_getfieldcheckint(L, 1, "dW"); + int dH = luaT_getfieldcheckint(L, 1, "dH"); + + THTensor *weight = luaT_getfieldcheckudata(L, 1, "weight", torch_Tensor); + THTensor *gradWeight = luaT_getfieldcheckudata(L, 1, "gradWeight", torch_Tensor); + THTensor *gradBias = luaT_getfieldcheckudata(L, 1, "gradBias", torch_Tensor); + + long nOutputPlane = weight->size[1]; + THArgCheck( nOutputPlane == gradOutput->size[input->nDimension == 4 ? 1 : 0], 1, "Number of output features is not equal to nOutputPlane" ); + + int dimw = 2; + int dimh = 1; + + if (input->nDimension == 4) + { + dimw++; + dimh++; + } + /* gradient to bias */ + real *gradBias_data = THTensor_(data)(gradBias); + real *gradOutput_data = THTensor_(data)(gradOutput); + long noutSlice = gradOutput->size[dimh]*gradOutput->size[dimw]; + /*THTensor* gradOutSlice = THTensor_(new)();*/ + + if (input->nDimension == 3) + { + long k; +#pragma omp parallel for private(k) + for(k = 0; k < nOutputPlane; k++) + { + /*THTensor_(select)(gradOutSlice, gradOutput, 0, k);*/ + real *ptr_gradOutput = gradOutput_data + k*noutSlice; + long l; + for(l = 0; l < noutSlice; l++) + gradBias_data[k] += scale*ptr_gradOutput[l]; + } + + /* gradient to kernels */ + THTensor_(conv2DRevger)(gradWeight, 1.0, scale, gradOutput, input, dH, dW); + } + else + { + long k; +#pragma omp parallel for private(k) + for(k = 0; k < nOutputPlane; k++) + { + long p; + for(p = 0; p < input->size[0]; p++) + { + /* BIAS */ + real *ptr_gradOutput = gradOutput_data + p*nOutputPlane*noutSlice + k*noutSlice; + long l; + for(l = 0; l < noutSlice; l++) + gradBias_data[k] += scale*ptr_gradOutput[l]; + } + } + /* gradient to kernels */ + THTensor_(conv2DRevgerm)(gradWeight, 1.0, scale, gradOutput, input, dH, dW); + } + return 0; +} + +static const struct luaL_Reg nn_(SpatialFullConvolution__) [] = { + {"SpatialFullConvolution_updateOutput", nn_(SpatialFullConvolution_updateOutput)}, + {"SpatialFullConvolution_updateGradInput", nn_(SpatialFullConvolution_updateGradInput)}, + {"SpatialFullConvolution_accGradParameters", nn_(SpatialFullConvolution_accGradParameters)}, + {NULL, NULL} +}; + +static void nn_(SpatialFullConvolution_init)(lua_State *L) +{ + luaT_pushmetatable(L, torch_Tensor); + luaT_registeratname(L, nn_(SpatialFullConvolution__), "nn"); + lua_pop(L,1); +} + +#endif diff --git a/generic/SpatialFullConvolutionMap.c b/generic/SpatialFullConvolutionMap.c new file mode 100644 index 0000000..8a5d9df --- /dev/null +++ b/generic/SpatialFullConvolutionMap.c @@ -0,0 +1,225 @@ +#ifndef TH_GENERIC_FILE +#define TH_GENERIC_FILE "generic/SpatialFullConvolutionMap.c" +#else + +static int nn_(SpatialFullConvolutionMap_updateOutput)(lua_State *L) +{ + THTensor *input = luaT_checkudata(L, 2, torch_Tensor); + int kW = luaT_getfieldcheckint(L, 1, "kW"); + int kH = luaT_getfieldcheckint(L, 1, "kH"); + int dW = luaT_getfieldcheckint(L, 1, "dW"); + int dH = luaT_getfieldcheckint(L, 1, "dH"); + int nInputPlane = luaT_getfieldcheckint(L, 1, "nInputPlane"); + int nOutputPlane = luaT_getfieldcheckint(L, 1, "nOutputPlane"); + + + THTensor *connTable = luaT_getfieldcheckudata(L, 1, "connTable", torch_Tensor); + THTensor *weight = luaT_getfieldcheckudata(L, 1, "weight", torch_Tensor); + THTensor *bias = luaT_getfieldcheckudata(L, 1, "bias", torch_Tensor); + THTensor *output = luaT_getfieldcheckudata(L, 1, "output", torch_Tensor); + + luaL_argcheck(L, input->nDimension == 3, 2, "3D tensor expected"); + luaL_argcheck(L, input->size[0] >= nInputPlane, 2, "invalid number of input planes"); + + + THTensor_(resize3d)(output, nOutputPlane, + (input->size[1] - 1) * dH + kH, + (input->size[2] - 1) * dW + kW); + + // contiguous + input = THTensor_(newContiguous)(input); + output = THTensor_(newContiguous)(output); + + // get raw pointers + real *input_data = THTensor_(data)(input); + real *output_data = THTensor_(data)(output); + real *weight_data = THTensor_(data)(weight); + real *bias_data = THTensor_(data)(bias); + real *connTable_data = THTensor_(data)(connTable); + + // and dims + long input_h = input->size[1]; + long input_w = input->size[2]; + long output_h = output->size[1]; + long output_w = output->size[2]; + long weight_h = weight->size[1]; + long weight_w = weight->size[2]; + + long p; +#pragma omp parallel for private(p) + for (p = 0; p < nOutputPlane; p++) { + // add bias + real *ptr_output = output_data + p*output_w*output_h; + long j; + for(j = 0; j < output_h*output_w; j++) + ptr_output[j] = bias_data[p]; + + // convolve all maps + int nweight = connTable->size[0]; + long k; + for (k = 0; k < nweight; k++) { + // get offsets for input/output + int o = (int)connTable_data[k*2+1]-1; + int i = (int)connTable_data[k*2+0]-1; + + if (o == p) + { + THTensor_(fullConv2Dptr)(output_data + o*output_w*output_h, + 1.0, + input_data + i*input_w*input_h, input_h, input_w, + weight_data + k*weight_w*weight_h, weight_h, weight_w, + dH, dW); + } + } + } + + // clean up + THTensor_(free)(input); + THTensor_(free)(output); + + return 1; +} + +static int nn_(SpatialFullConvolutionMap_updateGradInput)(lua_State *L) +{ + THTensor *input = luaT_checkudata(L, 2, torch_Tensor); + THTensor *gradOutput = luaT_checkudata(L, 3, torch_Tensor); + int dW = luaT_getfieldcheckint(L, 1, "dW"); + int dH = luaT_getfieldcheckint(L, 1, "dH"); + int nInputPlane = luaT_getfieldcheckint(L, 1, "nInputPlane"); + + THTensor *connTable = luaT_getfieldcheckudata(L, 1, "connTable", torch_Tensor); + THTensor *weight = luaT_getfieldcheckudata(L, 1, "weight", torch_Tensor); + THTensor *gradInput = luaT_getfieldcheckudata(L, 1, "gradInput", torch_Tensor); + + // contiguous + gradInput = THTensor_(newContiguous)(gradInput); + gradOutput = THTensor_(newContiguous)(gradOutput); + + // Resize/Zero + THTensor_(resizeAs)(gradInput, input); + THTensor_(zero)(gradInput); + + // get raw pointers + real *gradInput_data = THTensor_(data)(gradInput); + real *gradOutput_data = THTensor_(data)(gradOutput); + real *weight_data = THTensor_(data)(weight); + real *connTable_data = THTensor_(data)(connTable); + + // and dims + long input_h = input->size[1]; + long input_w = input->size[2]; + long output_h = gradOutput->size[1]; + long output_w = gradOutput->size[2]; + long weight_h = weight->size[1]; + long weight_w = weight->size[2]; + + long p; +#pragma omp parallel for private(p) + for(p = 0; p < nInputPlane; p++) + { + long k; + // backward all + int nkernel = connTable->size[0]; + for(k = 0; k < nkernel; k++) + { + int o = (int)connTable_data[k*2+1]-1; + int i = (int)connTable_data[k*2+0]-1; + if (i == p) + { + // gradient to input + THTensor_(validXCorr2Dptr)(gradInput_data + i*input_w*input_h, + 1.0, + gradOutput_data + o*output_w*output_h, output_h, output_w, + weight_data + k*weight_w*weight_h, weight_h, weight_w, + dH, dW); + } + } + } + + // clean up + THTensor_(free)(gradInput); + THTensor_(free)(gradOutput); + + return 1; +} + +static int nn_(SpatialFullConvolutionMap_accGradParameters)(lua_State *L) +{ + THTensor *input = luaT_checkudata(L, 2, torch_Tensor); + THTensor *gradOutput = luaT_checkudata(L, 3, torch_Tensor); + int dW = luaT_getfieldcheckint(L, 1, "dW"); + int dH = luaT_getfieldcheckint(L, 1, "dH"); + int nOutputPlane = luaT_getfieldcheckint(L, 1, "nOutputPlane"); + real scale = luaL_optnumber(L, 4, 1); + + THTensor *connTable = luaT_getfieldcheckudata(L, 1, "connTable", torch_Tensor); + THTensor *weight = luaT_getfieldcheckudata(L, 1, "weight", torch_Tensor); + THTensor *gradWeight = luaT_getfieldcheckudata(L, 1, "gradWeight", torch_Tensor); + THTensor *gradBias = luaT_getfieldcheckudata(L, 1, "gradBias", torch_Tensor); + + // contiguous + input = THTensor_(newContiguous)(input); + gradOutput = THTensor_(newContiguous)(gradOutput); + + // get raw pointers + real *input_data = THTensor_(data)(input); + real *gradOutput_data = THTensor_(data)(gradOutput); + real *gradWeight_data = THTensor_(data)(gradWeight); + real *gradBias_data = THTensor_(data)(gradBias); + + // and dims + long input_h = input->size[1]; + long input_w = input->size[2]; + long output_h = gradOutput->size[1]; + long output_w = gradOutput->size[2]; + long weight_h = weight->size[1]; + long weight_w = weight->size[2]; + + // gradients wrt bias + long k; +#pragma omp parallel for private(k) + for(k = 0; k < nOutputPlane; k++) { + real *ptr_gradOutput = gradOutput_data + k*output_w*output_h; + long l; + for(l = 0; l < output_h*output_w; l++) + gradBias_data[k] += scale*ptr_gradOutput[l]; + } + + // gradients wrt weight + int nkernel = connTable->size[0]; +#pragma omp parallel for private(k) + for(k = 0; k < nkernel; k++) + { + int o = (int)THTensor_(get2d)(connTable,k,1)-1; + int i = (int)THTensor_(get2d)(connTable,k,0)-1; + + // gradient to kernel + THTensor_(validXCorr2DRevptr)(gradWeight_data + k*weight_w*weight_h, + scale, + gradOutput_data + o*output_w*output_h, output_h, output_w, + input_data + i*input_w*input_h, input_h, input_w, + dH, dW); + } + + // clean up + THTensor_(free)(input); + THTensor_(free)(gradOutput); + return 0; +} + +static const struct luaL_Reg nn_(SpatialFullConvolutionMapStuff__) [] = { + {"SpatialFullConvolutionMap_updateOutput", nn_(SpatialFullConvolutionMap_updateOutput)}, + {"SpatialFullConvolutionMap_updateGradInput", nn_(SpatialFullConvolutionMap_updateGradInput)}, + {"SpatialFullConvolutionMap_accGradParameters", nn_(SpatialFullConvolutionMap_accGradParameters)}, + {NULL, NULL} +}; + +static void nn_(SpatialFullConvolutionMap_init)(lua_State *L) +{ + luaT_pushmetatable(L, torch_Tensor); + luaT_registeratname(L, nn_(SpatialFullConvolutionMapStuff__), "nn"); + lua_pop(L,1); +} + +#endif -- cgit v1.2.3