diff options
Diffstat (limited to 'intern/cycles/device')
-rw-r--r-- | intern/cycles/device/device_cpu.cpp | 13 | ||||
-rw-r--r-- | intern/cycles/device/device_cuda.cpp | 27 | ||||
-rw-r--r-- | intern/cycles/device/device_denoising.cpp | 52 | ||||
-rw-r--r-- | intern/cycles/device/device_denoising.h | 21 | ||||
-rw-r--r-- | intern/cycles/device/opencl/opencl_base.cpp | 48 |
5 files changed, 62 insertions, 99 deletions
diff --git a/intern/cycles/device/device_cpu.cpp b/intern/cycles/device/device_cpu.cpp index 6867dffc727..e92bbbfa6e6 100644 --- a/intern/cycles/device/device_cpu.cpp +++ b/intern/cycles/device/device_cpu.cpp @@ -471,9 +471,10 @@ public: int w = align_up(rect.z-rect.x, 4); int h = rect.w-rect.y; - float *blurDifference = (float*) task->nlm_state.temporary_1_ptr; - float *difference = (float*) task->nlm_state.temporary_2_ptr; - float *weightAccum = (float*) task->nlm_state.temporary_3_ptr; + float *temporary_mem = (float*) task->buffer.temporary_mem.device_pointer; + float *blurDifference = temporary_mem; + float *difference = temporary_mem + task->buffer.pass_stride; + float *weightAccum = temporary_mem + 2*task->buffer.pass_stride; memset(weightAccum, 0, sizeof(float)*w*h); memset((float*) out_ptr, 0, sizeof(float)*w*h); @@ -537,8 +538,9 @@ public: mem_zero(task->storage.XtWX); mem_zero(task->storage.XtWY); - float *difference = (float*) task->reconstruction_state.temporary_1_ptr; - float *blurDifference = (float*) task->reconstruction_state.temporary_2_ptr; + float *temporary_mem = (float*) task->buffer.temporary_mem.device_pointer; + float *difference = temporary_mem; + float *blurDifference = temporary_mem + task->buffer.pass_stride; int r = task->radius; for(int i = 0; i < (2*r+1)*(2*r+1); i++) { @@ -713,6 +715,7 @@ public: denoising.filter_area = make_int4(tile.x, tile.y, tile.w, tile.h); denoising.render_buffer.samples = tile.sample; + denoising.buffer.gpu_temporary_mem = false; denoising.run_denoising(&tile); } diff --git a/intern/cycles/device/device_cuda.cpp b/intern/cycles/device/device_cuda.cpp index 73c9697223d..da8e49f129f 100644 --- a/intern/cycles/device/device_cuda.cpp +++ b/intern/cycles/device/device_cuda.cpp @@ -1294,23 +1294,19 @@ public: float a = task->nlm_state.a; float k_2 = task->nlm_state.k_2; - int shift_stride = stride*h; + int pass_stride = task->buffer.pass_stride; int num_shifts = (2*r+1)*(2*r+1); - int mem_size = sizeof(float)*shift_stride*num_shifts; int channel_offset = 0; - device_only_memory<uchar> temporary_mem(this, "Denoising temporary_mem"); - temporary_mem.alloc_to_device(2*mem_size); - if(have_error()) return false; - CUdeviceptr difference = cuda_device_ptr(temporary_mem.device_pointer); - CUdeviceptr blurDifference = difference + mem_size; + CUdeviceptr difference = cuda_device_ptr(task->buffer.temporary_mem.device_pointer); + CUdeviceptr blurDifference = difference + sizeof(float)*pass_stride*num_shifts; + CUdeviceptr weightAccum = difference + 2*sizeof(float)*pass_stride*num_shifts; - CUdeviceptr weightAccum = task->nlm_state.temporary_3_ptr; - cuda_assert(cuMemsetD8(weightAccum, 0, sizeof(float)*shift_stride)); - cuda_assert(cuMemsetD8(out_ptr, 0, sizeof(float)*shift_stride)); + cuda_assert(cuMemsetD8(weightAccum, 0, sizeof(float)*pass_stride)); + cuda_assert(cuMemsetD8(out_ptr, 0, sizeof(float)*pass_stride)); { CUfunction cuNLMCalcDifference, cuNLMBlur, cuNLMCalcWeight, cuNLMUpdateOutput; @@ -1326,10 +1322,10 @@ public: CUDA_GET_BLOCKSIZE_1D(cuNLMCalcDifference, w*h, num_shifts); - void *calc_difference_args[] = {&guide_ptr, &variance_ptr, &difference, &w, &h, &stride, &shift_stride, &r, &channel_offset, &a, &k_2}; - void *blur_args[] = {&difference, &blurDifference, &w, &h, &stride, &shift_stride, &r, &f}; - void *calc_weight_args[] = {&blurDifference, &difference, &w, &h, &stride, &shift_stride, &r, &f}; - void *update_output_args[] = {&blurDifference, &image_ptr, &out_ptr, &weightAccum, &w, &h, &stride, &shift_stride, &r, &f}; + void *calc_difference_args[] = {&guide_ptr, &variance_ptr, &difference, &w, &h, &stride, &pass_stride, &r, &channel_offset, &a, &k_2}; + void *blur_args[] = {&difference, &blurDifference, &w, &h, &stride, &pass_stride, &r, &f}; + void *calc_weight_args[] = {&blurDifference, &difference, &w, &h, &stride, &pass_stride, &r, &f}; + void *update_output_args[] = {&blurDifference, &image_ptr, &out_ptr, &weightAccum, &w, &h, &stride, &pass_stride, &r, &f}; CUDA_LAUNCH_KERNEL_1D(cuNLMCalcDifference, calc_difference_args); CUDA_LAUNCH_KERNEL_1D(cuNLMBlur, blur_args); @@ -1338,8 +1334,6 @@ public: CUDA_LAUNCH_KERNEL_1D(cuNLMUpdateOutput, update_output_args); } - temporary_mem.free(); - { CUfunction cuNLMNormalize; cuda_assert(cuModuleGetFunction(&cuNLMNormalize, cuFilterModule, "kernel_cuda_filter_nlm_normalize")); @@ -1614,6 +1608,7 @@ public: denoising.filter_area = make_int4(rtile.x, rtile.y, rtile.w, rtile.h); denoising.render_buffer.samples = rtile.sample; + denoising.buffer.gpu_temporary_mem = true; denoising.run_denoising(&rtile); } diff --git a/intern/cycles/device/device_denoising.cpp b/intern/cycles/device/device_denoising.cpp index c0d4634262d..23c18fa15b2 100644 --- a/intern/cycles/device/device_denoising.cpp +++ b/intern/cycles/device/device_denoising.cpp @@ -51,10 +51,8 @@ DenoisingTask::~DenoisingTask() storage.XtWY.free(); storage.transform.free(); storage.rank.free(); - storage.temporary_1.free(); - storage.temporary_2.free(); - storage.temporary_color.free(); buffer.mem.free(); + buffer.temporary_mem.free(); tile_info_mem.free(); } @@ -99,6 +97,16 @@ void DenoisingTask::setup_denoising_buffer() /* Pad the total size by four floats since the SIMD kernels might go a bit over the end. */ int mem_size = align_up(buffer.pass_stride * buffer.passes + 4, alignment_floats); buffer.mem.alloc_to_device(mem_size, false); + + /* CPUs process shifts sequentially while GPUs process them in parallel. */ + int num_shifts = 1; + if(buffer.gpu_temporary_mem) { + /* Shadowing prefiltering uses a radius of 6, so allocate at least that much. */ + int max_radius = max(radius, 6); + num_shifts = (2*max_radius + 1) * (2*max_radius + 1); + } + /* Allocate two layers per shift as well as one for the weight accumulation. */ + buffer.temporary_mem.alloc_to_device((2*num_shifts + 1) * buffer.pass_stride); } void DenoisingTask::prefilter_shadowing() @@ -111,13 +119,6 @@ void DenoisingTask::prefilter_shadowing() device_sub_ptr sample_var_var (buffer.mem, 3*buffer.pass_stride, buffer.pass_stride); device_sub_ptr buffer_var (buffer.mem, 5*buffer.pass_stride, buffer.pass_stride); device_sub_ptr filtered_var (buffer.mem, 6*buffer.pass_stride, buffer.pass_stride); - device_sub_ptr nlm_temporary_1(buffer.mem, 7*buffer.pass_stride, buffer.pass_stride); - device_sub_ptr nlm_temporary_2(buffer.mem, 8*buffer.pass_stride, buffer.pass_stride); - device_sub_ptr nlm_temporary_3(buffer.mem, 9*buffer.pass_stride, buffer.pass_stride); - - nlm_state.temporary_1_ptr = *nlm_temporary_1; - nlm_state.temporary_2_ptr = *nlm_temporary_2; - nlm_state.temporary_3_ptr = *nlm_temporary_3; /* Get the A/B unfiltered passes, the combined sample variance, the estimated variance of the sample variance and the buffer variance. */ functions.divide_shadow(*unfiltered_a, *unfiltered_b, *sample_var, *sample_var_var, *buffer_var); @@ -154,13 +155,6 @@ void DenoisingTask::prefilter_features() { device_sub_ptr unfiltered (buffer.mem, 8*buffer.pass_stride, buffer.pass_stride); device_sub_ptr variance (buffer.mem, 9*buffer.pass_stride, buffer.pass_stride); - device_sub_ptr nlm_temporary_1(buffer.mem, 10*buffer.pass_stride, buffer.pass_stride); - device_sub_ptr nlm_temporary_2(buffer.mem, 11*buffer.pass_stride, buffer.pass_stride); - device_sub_ptr nlm_temporary_3(buffer.mem, 12*buffer.pass_stride, buffer.pass_stride); - - nlm_state.temporary_1_ptr = *nlm_temporary_1; - nlm_state.temporary_2_ptr = *nlm_temporary_2; - nlm_state.temporary_3_ptr = *nlm_temporary_3; int mean_from[] = { 0, 1, 2, 12, 6, 7, 8 }; int variance_from[] = { 3, 4, 5, 13, 9, 10, 11}; @@ -183,17 +177,11 @@ void DenoisingTask::prefilter_color() int variance_to[] = {11, 12, 13}; int num_color_passes = 3; - storage.temporary_color.alloc_to_device(3*buffer.pass_stride, false); - device_sub_ptr nlm_temporary_1(storage.temporary_color, 0*buffer.pass_stride, buffer.pass_stride); - device_sub_ptr nlm_temporary_2(storage.temporary_color, 1*buffer.pass_stride, buffer.pass_stride); - device_sub_ptr nlm_temporary_3(storage.temporary_color, 2*buffer.pass_stride, buffer.pass_stride); - - nlm_state.temporary_1_ptr = *nlm_temporary_1; - nlm_state.temporary_2_ptr = *nlm_temporary_2; - nlm_state.temporary_3_ptr = *nlm_temporary_3; + device_only_memory<float> temporary_color(device, "denoising temporary color"); + temporary_color.alloc_to_device(3*buffer.pass_stride, false); for(int pass = 0; pass < num_color_passes; pass++) { - device_sub_ptr color_pass(storage.temporary_color, pass*buffer.pass_stride, buffer.pass_stride); + device_sub_ptr color_pass(temporary_color, pass*buffer.pass_stride, buffer.pass_stride); device_sub_ptr color_var_pass(buffer.mem, variance_to[pass]*buffer.pass_stride, buffer.pass_stride); functions.get_feature(mean_from[pass], variance_from[pass], *color_pass, *color_var_pass); } @@ -201,9 +189,7 @@ void DenoisingTask::prefilter_color() device_sub_ptr depth_pass (buffer.mem, 0, buffer.pass_stride); device_sub_ptr color_var_pass(buffer.mem, variance_to[0]*buffer.pass_stride, 3*buffer.pass_stride); device_sub_ptr output_pass (buffer.mem, mean_to[0]*buffer.pass_stride, 3*buffer.pass_stride); - functions.detect_outliers(storage.temporary_color.device_pointer, *color_var_pass, *depth_pass, *output_pass); - - storage.temporary_color.free(); + functions.detect_outliers(temporary_color.device_pointer, *color_var_pass, *depth_pass, *output_pass); } void DenoisingTask::construct_transform() @@ -219,14 +205,6 @@ void DenoisingTask::construct_transform() void DenoisingTask::reconstruct() { - - device_only_memory<float> temporary_1(device, "Denoising NLM temporary 1"); - device_only_memory<float> temporary_2(device, "Denoising NLM temporary 2"); - temporary_1.alloc_to_device(buffer.pass_stride, false); - temporary_2.alloc_to_device(buffer.pass_stride, false); - reconstruction_state.temporary_1_ptr = temporary_1.device_pointer; - reconstruction_state.temporary_2_ptr = temporary_2.device_pointer; - storage.XtWX.alloc_to_device(storage.w*storage.h*XTWX_SIZE, false); storage.XtWY.alloc_to_device(storage.w*storage.h*XTWY_SIZE, false); diff --git a/intern/cycles/device/device_denoising.h b/intern/cycles/device/device_denoising.h index e0da52867f1..7474f71ff78 100644 --- a/intern/cycles/device/device_denoising.h +++ b/intern/cycles/device/device_denoising.h @@ -96,9 +96,6 @@ public: /* Stores state of the current Reconstruction operation, * which is accessed by the device in order to perform the operation. */ struct ReconstructionState { - device_ptr temporary_1_ptr; /* There two images are used as temporary storage. */ - device_ptr temporary_2_ptr; - int4 filter_window; int4 buffer_params; @@ -109,10 +106,6 @@ public: /* Stores state of the current NLM operation, * which is accessed by the device in order to perform the operation. */ struct NLMState { - device_ptr temporary_1_ptr; /* There three images are used as temporary storage. */ - device_ptr temporary_2_ptr; - device_ptr temporary_3_ptr; - int r; /* Search radius of the filter. */ int f; /* Patch size of the filter. */ float a; /* Variance compensation factor in the MSE estimation. */ @@ -126,9 +119,6 @@ public: device_only_memory<int> rank; device_only_memory<float> XtWX; device_only_memory<float3> XtWY; - device_only_memory<float> temporary_1; - device_only_memory<float> temporary_2; - device_only_memory<float> temporary_color; int w; int h; @@ -136,10 +126,7 @@ public: : transform(device, "denoising transform"), rank(device, "denoising rank"), XtWX(device, "denoising XtWX"), - XtWY(device, "denoising XtWY"), - temporary_1(device, "denoising NLM temporary 1"), - temporary_2(device, "denoising NLM temporary 2"), - temporary_color(device, "denoising temporary color") + XtWY(device, "denoising XtWY") {} } storage; @@ -155,9 +142,13 @@ public: int h; int width; device_only_memory<float> mem; + device_only_memory<float> temporary_mem; + + bool gpu_temporary_mem; DenoiseBuffers(Device *device) - : mem(device, "denoising pixel buffer") + : mem(device, "denoising pixel buffer"), + temporary_mem(device, "denoising temporary mem") {} } buffer; diff --git a/intern/cycles/device/opencl/opencl_base.cpp b/intern/cycles/device/opencl/opencl_base.cpp index 9a50d217321..69a2bd7adcb 100644 --- a/intern/cycles/device/opencl/opencl_base.cpp +++ b/intern/cycles/device/opencl/opencl_base.cpp @@ -738,7 +738,6 @@ bool OpenCLDeviceBase::denoising_non_local_means(device_ptr image_ptr, device_ptr out_ptr, DenoisingTask *task) { - int stride = task->buffer.stride; int w = task->buffer.width; int h = task->buffer.h; @@ -747,24 +746,23 @@ bool OpenCLDeviceBase::denoising_non_local_means(device_ptr image_ptr, float a = task->nlm_state.a; float k_2 = task->nlm_state.k_2; - int shift_stride = stride*h; + int pass_stride = task->buffer.pass_stride; int num_shifts = (2*r+1)*(2*r+1); - int mem_size = sizeof(float)*shift_stride*num_shifts; - cl_mem weightAccum = CL_MEM_PTR(task->nlm_state.temporary_3_ptr); - - cl_mem difference = clCreateBuffer(cxContext, CL_MEM_READ_WRITE, mem_size, NULL, &ciErr); - opencl_assert_err(ciErr, "clCreateBuffer denoising_non_local_means"); - cl_mem blurDifference = clCreateBuffer(cxContext, CL_MEM_READ_WRITE, mem_size, NULL, &ciErr); - opencl_assert_err(ciErr, "clCreateBuffer denoising_non_local_means"); + device_sub_ptr difference(task->buffer.temporary_mem, 0, pass_stride*num_shifts); + device_sub_ptr blurDifference(task->buffer.temporary_mem, pass_stride*num_shifts, pass_stride*num_shifts); + device_sub_ptr weightAccum(task->buffer.temporary_mem, 2*pass_stride*num_shifts, pass_stride); + cl_mem weightAccum_mem = CL_MEM_PTR(*weightAccum); + cl_mem difference_mem = CL_MEM_PTR(*difference); + cl_mem blurDifference_mem = CL_MEM_PTR(*blurDifference); cl_mem image_mem = CL_MEM_PTR(image_ptr); cl_mem guide_mem = CL_MEM_PTR(guide_ptr); cl_mem variance_mem = CL_MEM_PTR(variance_ptr); cl_mem out_mem = CL_MEM_PTR(out_ptr); - mem_zero_kernel(task->nlm_state.temporary_3_ptr, sizeof(float)*w*h); - mem_zero_kernel(out_ptr, sizeof(float)*w*h); + mem_zero_kernel(*difference, sizeof(float)*pass_stride); + mem_zero_kernel(out_ptr, sizeof(float)*pass_stride); cl_kernel ckNLMCalcDifference = denoising_program(ustring("filter_nlm_calc_difference")); cl_kernel ckNLMBlur = denoising_program(ustring("filter_nlm_blur")); @@ -775,29 +773,29 @@ bool OpenCLDeviceBase::denoising_non_local_means(device_ptr image_ptr, kernel_set_args(ckNLMCalcDifference, 0, guide_mem, variance_mem, - difference, + difference_mem, w, h, stride, - shift_stride, + pass_stride, r, 0, a, k_2); kernel_set_args(ckNLMBlur, 0, - difference, - blurDifference, + difference_mem, + blurDifference_mem, w, h, stride, - shift_stride, + pass_stride, r, f); kernel_set_args(ckNLMCalcWeight, 0, - blurDifference, - difference, + blurDifference_mem, + difference_mem, w, h, stride, - shift_stride, + pass_stride, r, f); kernel_set_args(ckNLMUpdateOutput, 0, - blurDifference, + blurDifference_mem, image_mem, out_mem, - weightAccum, + weightAccum_mem, w, h, stride, - shift_stride, + pass_stride, r, f); enqueue_kernel(ckNLMCalcDifference, w*h, num_shifts, true); @@ -806,11 +804,8 @@ bool OpenCLDeviceBase::denoising_non_local_means(device_ptr image_ptr, enqueue_kernel(ckNLMBlur, w*h, num_shifts, true); enqueue_kernel(ckNLMUpdateOutput, w*h, num_shifts, true); - opencl_assert(clReleaseMemObject(difference)); - opencl_assert(clReleaseMemObject(blurDifference)); - kernel_set_args(ckNLMNormalize, 0, - out_mem, weightAccum, w, h, stride); + out_mem, weightAccum_mem, w, h, stride); enqueue_kernel(ckNLMNormalize, w, h); return true; @@ -1081,6 +1076,7 @@ void OpenCLDeviceBase::denoise(RenderTile &rtile, DenoisingTask& denoising) denoising.filter_area = make_int4(rtile.x, rtile.y, rtile.w, rtile.h); denoising.render_buffer.samples = rtile.sample; + denoising.buffer.gpu_temporary_mem = true; denoising.run_denoising(&rtile); } |