Sequencer: per-sequence modifier stack for color grading

This implements basic color grading modifiers in sequencer, supporting
color balance, RGB curves and HUE corrections.

Implementation is close to object modifiers, some details are there:

http://wiki.blender.org/index.php/User:Nazg-gul/SequencerModifiers

Modifiers supports multi-threaded calculation, masks and instant
parameter changes.

Also added cache for pre-processed image buffers for current frame,
so changing sequence properties does not require rendering of original
sequence (like rendering scene, loading file from disk and so)

1 files changed, 522 insertions, 0 deletions

diff --git a/source/blender/blenkernel/intern/seqmodifier.c b/source/blender/blenkernel/intern/seqmodifier.c
new file mode 100644
index 00000000000..4157a3a1562
--- /dev/null
+++ b/source/blender/blenkernel/intern/seqmodifier.c
@@ -0,0 +1,522 @@
+/*
+ * ***** BEGIN GPL LICENSE BLOCK *****
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ * The Original Code is Copyright (C) 2012 Blender Foundation.
+ * All rights reserved.
+ *
+ * Contributor(s): Blender Foundation,
+ *                 Sergey Sharybin
+ *
+ * ***** END GPL LICENSE BLOCK *****
+ */
+
+/** \file blender/blenkernel/intern/seqmodifier.c
+ *  \ingroup bke
+ */
+
+#include <stddef.h>
+#include <string.h>
+
+#include "MEM_guardedalloc.h"
+
+#include "BLI_listbase.h"
+#include "BLI_path_util.h"
+#include "BLI_string.h"
+#include "BLI_utildefines.h"
+#include "BLI_math.h"
+
+#include "DNA_sequence_types.h"
+
+#include "BKE_colortools.h"
+#include "BKE_sequencer.h"
+#include "BKE_utildefines.h"
+
+#include "IMB_imbuf.h"
+#include "IMB_imbuf_types.h"
+
+static SequenceModifierTypeInfo *modifiersTypes[NUM_SEQUENCE_MODIFIER_TYPES];
+static int modifierTypesInit = FALSE;
+
+/*********************** Modifiers *************************/
+
+typedef void (*modifier_apply_threaded_cb) (int width, int height, unsigned char *rect, float *rect_float,
+                                            unsigned char *mask_rect, float *mask_rect_float, void *data_v);
+
+typedef struct ModifierInitData {
+	ImBuf *ibuf;
+	ImBuf *mask;
+	void *user_data;
+
+	modifier_apply_threaded_cb apply_callback;
+} ModifierInitData;
+
+typedef struct ModifierThread {
+	int width, height;
+
+	unsigned char *rect, *mask_rect;
+	float *rect_float, *mask_rect_float;
+
+	void *user_data;
+
+	modifier_apply_threaded_cb apply_callback;
+} ModifierThread;
+
+
+static ImBuf *modifier_mask_get(SequenceModifierData *smd, SeqRenderData context, int cfra, int make_float)
+{
+	return BKE_sequencer_render_mask_input(context, smd->mask_input_type, smd->mask_sequence, smd->mask_id, cfra, make_float);
+}
+
+static void modifier_init_handle(void *handle_v, int start_line, int tot_line, void *init_data_v)
+{
+	ModifierThread *handle = (ModifierThread *) handle_v;
+	ModifierInitData *init_data = (ModifierInitData *) init_data_v;
+	ImBuf *ibuf = init_data->ibuf;
+	ImBuf *mask = init_data->mask;
+
+	int offset = 4 * start_line * ibuf->x;
+
+	memset(handle, 0, sizeof(ModifierThread));
+
+	handle->width = ibuf->x;
+	handle->height = tot_line;
+	handle->apply_callback = init_data->apply_callback;
+	handle->user_data = init_data->user_data;
+
+	if (ibuf->rect)
+		handle->rect = (unsigned char *) ibuf->rect + offset;
+
+	if (ibuf->rect_float)
+		handle->rect_float = ibuf->rect_float + offset;
+
+	if (mask) {
+		if (mask->rect)
+			handle->mask_rect = (unsigned char *) mask->rect + offset;
+
+		if (mask->rect_float)
+			handle->mask_rect_float = mask->rect_float + offset;
+	}
+	else {
+		handle->mask_rect = NULL;
+		handle->mask_rect_float = NULL;
+	}
+}
+
+static void *modifier_do_thread(void *thread_data_v)
+{
+	ModifierThread *td = (ModifierThread *) thread_data_v;
+
+	td->apply_callback(td->width, td->height, td->rect, td->rect_float, td->mask_rect, td->mask_rect_float, td->user_data);
+
+	return NULL;
+}
+
+static void modifier_apply_threaded(ImBuf *ibuf, ImBuf *mask, modifier_apply_threaded_cb apply_callback, void *user_data)
+{
+	ModifierInitData init_data;
+
+	init_data.ibuf = ibuf;
+	init_data.mask = mask;
+	init_data.user_data = user_data;
+
+	init_data.apply_callback = apply_callback;
+
+	IMB_processor_apply_threaded(ibuf->y, sizeof(ModifierThread), &init_data,
+                                 modifier_init_handle, modifier_do_thread);
+}
+
+/* **** Color Balance Modifier **** */
+
+void colorBalance_init_data(SequenceModifierData *smd)
+{
+	ColorBalanceModifierData *cbmd = (ColorBalanceModifierData *) smd;
+	int c;
+
+	cbmd->color_multiply = 1.0f;
+
+	for (c = 0; c < 3; c++) {
+		cbmd->color_balance.lift[c] = 1.0f;
+		cbmd->color_balance.gamma[c] = 1.0f;
+		cbmd->color_balance.gain[c] = 1.0f;
+	}
+}
+
+ImBuf *colorBalance_apply(SequenceModifierData *smd, ImBuf *ibuf, ImBuf *mask)
+{
+	ColorBalanceModifierData *cbmd = (ColorBalanceModifierData *) smd;
+	ImBuf *ibuf_new = IMB_dupImBuf(ibuf);
+
+	BKE_sequencer_color_balance_apply(&cbmd->color_balance, ibuf_new, cbmd->color_multiply, FALSE, mask);
+
+	return ibuf_new;
+}
+
+static SequenceModifierTypeInfo seqModifier_ColorBalance = {
+	"Color Balance",                   /* name */
+	"ColorBalanceModifierData",        /* struct_name */
+	sizeof(ColorBalanceModifierData),  /* struct_size */
+	colorBalance_init_data,            /* init_data */
+	NULL,                              /* free_data */
+	NULL,                              /* copy_data */
+	colorBalance_apply                 /* apply */
+};
+
+/* **** Curves Modifier **** */
+
+void curves_init_data(SequenceModifierData *smd)
+{
+	CurvesModifierData *cmd = (CurvesModifierData *) smd;
+
+	curvemapping_set_defaults(&cmd->curve_mapping, 4, 0.0f, 0.0f, 1.0f, 1.0f);
+}
+
+void curves_free_data(SequenceModifierData *smd)
+{
+	CurvesModifierData *cmd = (CurvesModifierData *) smd;
+
+	curvemapping_free_data(&cmd->curve_mapping);
+}
+
+void curves_copy_data(SequenceModifierData *target, SequenceModifierData *smd)
+{
+	CurvesModifierData *cmd = (CurvesModifierData *) smd;
+	CurvesModifierData *cmd_target = (CurvesModifierData *) target;
+
+	curvemapping_copy_data(&cmd_target->curve_mapping, &cmd->curve_mapping);
+}
+
+void curves_apply_threaded(int width, int height, unsigned char *rect, float *rect_float,
+                           unsigned char *mask_rect, float *mask_rect_float, void *data_v)
+{
+	CurveMapping *curve_mapping = (CurveMapping *) data_v;
+	int x, y;
+
+	for (y = 0; y < height; y++) {
+		for (x = 0; x < width; x++) {
+			int pixel_index = (y * width + x) * 4;
+
+			if (rect_float) {
+				float *pixel = rect_float + pixel_index;
+				float result[3];
+
+				curvemapping_evaluate_premulRGBF(curve_mapping, result, pixel);
+
+				if (mask_rect_float) {
+					float *m = mask_rect_float + pixel_index;
+
+					pixel[0] = pixel[0] * (1.0f - m[0]) + result[0] * m[0];
+					pixel[1] = pixel[1] * (1.0f - m[1]) + result[1] * m[1];
+					pixel[2] = pixel[2] * (1.0f - m[2]) + result[2] * m[2];
+				}
+				else {
+					pixel[0] = result[0];
+					pixel[1] = result[1];
+					pixel[2] = result[2];
+				}
+			}
+			if (rect) {
+				unsigned char *pixel = rect + pixel_index;
+				unsigned char result[3];
+
+				curvemapping_evaluate_premulRGB(curve_mapping, result, pixel);
+
+				if (mask_rect) {
+					float t[3];
+
+					rgb_uchar_to_float(t, mask_rect + pixel_index);
+
+					pixel[0] = pixel[0] * (1.0f - t[0]) + result[0] * t[0];
+					pixel[1] = pixel[1] * (1.0f - t[1]) + result[1] * t[1];
+					pixel[2] = pixel[2] * (1.0f - t[2]) + result[2] * t[2];
+				}
+				else {
+					pixel[0] = result[0];
+					pixel[1] = result[1];
+					pixel[2] = result[2];
+				}
+			}
+		}
+	}
+}
+
+ImBuf *curves_apply(struct SequenceModifierData *smd, ImBuf *ibuf, ImBuf *mask)
+{
+	CurvesModifierData *cmd = (CurvesModifierData *) smd;
+	ImBuf *ibuf_new = IMB_dupImBuf(ibuf);
+
+	float black[3] = {0.0f, 0.0f, 0.0f};
+	float white[3] = {1.0f, 1.0f, 1.0f};
+
+	curvemapping_initialize(&cmd->curve_mapping);
+
+	curvemapping_premultiply(&cmd->curve_mapping, 0);
+	curvemapping_set_black_white(&cmd->curve_mapping, black, white);
+
+	modifier_apply_threaded(ibuf_new, mask, curves_apply_threaded, &cmd->curve_mapping);
+
+	curvemapping_premultiply(&cmd->curve_mapping, 1);
+
+	return ibuf_new;
+}
+
+static SequenceModifierTypeInfo seqModifier_Curves = {
+	"Curves",                    /* name */
+	"CurvesModifierData",        /* struct_name */
+	sizeof(CurvesModifierData),  /* struct_size */
+	curves_init_data,            /* init_data */
+	curves_free_data,            /* free_data */
+	curves_copy_data,            /* copy_data */
+	curves_apply                 /* apply */
+};
+
+/* **** Hue Correct Modifier **** */
+
+void hue_correct_init_data(SequenceModifierData *smd)
+{
+	HueCorrectModifierData *hcmd = (HueCorrectModifierData *) smd;
+	int c;
+
+	curvemapping_set_defaults(&hcmd->curve_mapping, 1, 0.0f, 0.0f, 1.0f, 1.0f);
+	hcmd->curve_mapping.preset = CURVE_PRESET_MID9;
+
+	for (c = 0; c < 3; c++) {
+		CurveMap *cuma = &hcmd->curve_mapping.cm[c];
+
+		curvemap_reset(cuma, &hcmd->curve_mapping.clipr, hcmd->curve_mapping.preset, CURVEMAP_SLOPE_POSITIVE);
+	}
+
+	/* default to showing Saturation */
+	hcmd->curve_mapping.cur = 1;
+}
+
+void hue_correct_free_data(SequenceModifierData *smd)
+{
+	HueCorrectModifierData *hcmd = (HueCorrectModifierData *) smd;
+
+	curvemapping_free_data(&hcmd->curve_mapping);
+}
+
+void hue_correct_copy_data(SequenceModifierData *target, SequenceModifierData *smd)
+{
+	HueCorrectModifierData *hcmd = (HueCorrectModifierData *) smd;
+	HueCorrectModifierData *hcmd_target = (HueCorrectModifierData *) target;
+
+	curvemapping_copy_data(&hcmd_target->curve_mapping, &hcmd->curve_mapping);
+}
+
+void hue_correct_apply_threaded(int width, int height, unsigned char *rect, float *rect_float,
+                                unsigned char *mask_rect, float *mask_rect_float, void *data_v)
+{
+	CurveMapping *curve_mapping = (CurveMapping *) data_v;
+	int x, y;
+
+	for (y = 0; y < height; y++) {
+		for (x = 0; x < width; x++) {
+			int pixel_index = (y * width + x) * 4;
+			float pixel[3], result[3], mask[3] = {1.0f, 1.0f, 1.0f};
+			float hsv[3], f;
+
+			if (rect_float)
+				copy_v3_v3(pixel, rect_float + pixel_index);
+			else
+				rgb_uchar_to_float(pixel, rect + pixel_index);
+
+			rgb_to_hsv(pixel[0], pixel[1], pixel[2], hsv, hsv + 1, hsv + 2);
+
+			/* adjust hue, scaling returned default 0.5 up to 1 */
+			f = curvemapping_evaluateF(curve_mapping, 0, hsv[0]);
+			hsv[0] += f - 0.5f;
+
+			/* adjust saturation, scaling returned default 0.5 up to 1 */
+			f = curvemapping_evaluateF(curve_mapping, 1, hsv[0]);
+			hsv[1] *= (f * 2.0f);
+
+			/* adjust value, scaling returned default 0.5 up to 1 */
+			f = curvemapping_evaluateF(curve_mapping, 2, hsv[0]);
+			hsv[2] *= (f * 2.f);
+
+			hsv[0] = hsv[0] - floorf(hsv[0]); /* mod 1.0 */
+			CLAMP(hsv[1], 0.0f, 1.0f);
+
+			/* convert back to rgb */
+			hsv_to_rgb(hsv[0], hsv[1], hsv[2], result, result + 1, result + 2);
+
+			if (mask_rect_float)
+					copy_v3_v3(mask, mask_rect_float + pixel_index);
+			else if (mask_rect)
+					rgb_uchar_to_float(mask, mask_rect + pixel_index);
+
+			result[0] = pixel[0] * (1.0f - mask[0]) + result[0] * mask[0];
+			result[1] = pixel[1] * (1.0f - mask[1]) + result[1] * mask[1];
+			result[2] = pixel[2] * (1.0f - mask[2]) + result[2] * mask[2];
+
+			if (rect_float)
+				copy_v3_v3(rect_float + pixel_index, result);
+			else
+				rgb_float_to_uchar(rect + pixel_index, result);
+		}
+	}
+}
+
+ImBuf *hue_correct_apply(struct SequenceModifierData *smd, ImBuf *ibuf, ImBuf *mask)
+{
+	HueCorrectModifierData *hcmd = (HueCorrectModifierData *) smd;
+	ImBuf *ibuf_new = IMB_dupImBuf(ibuf);
+
+	curvemapping_initialize(&hcmd->curve_mapping);
+
+	modifier_apply_threaded(ibuf_new, mask, hue_correct_apply_threaded, &hcmd->curve_mapping);
+
+	return ibuf_new;
+}
+
+static SequenceModifierTypeInfo seqModifier_HueCorrect = {
+	"Hue Correct",                    /* name */
+	"HueCorrectModifierData",         /* struct_name */
+	sizeof(HueCorrectModifierData),   /* struct_size */
+	hue_correct_init_data,            /* init_data */
+	hue_correct_free_data,            /* free_data */
+	hue_correct_copy_data,            /* copy_data */
+	hue_correct_apply                 /* apply */
+};
+
+/*********************** Modifier functions *************************/
+
+static void sequence_modifier_type_info_init(void)
+{
+#define INIT_TYPE(typeName) (modifiersTypes[seqModifierType_##typeName] = &seqModifier_##typeName)
+
+	INIT_TYPE(ColorBalance);
+	INIT_TYPE(Curves);
+	INIT_TYPE(HueCorrect);
+
+#undef INIT_TYPE
+}
+
+SequenceModifierTypeInfo *BKE_sequence_modifier_type_info_get(int type)
+{
+	if (!modifierTypesInit) {
+		sequence_modifier_type_info_init();
+		modifierTypesInit = TRUE;
+	}
+
+	return modifiersTypes[type];
+}
+
+void BKE_sequence_modifier_new(Sequence *seq, int type)
+{
+	SequenceModifierData *smd;
+	SequenceModifierTypeInfo *smti = BKE_sequence_modifier_type_info_get(type);
+
+	smd = MEM_callocN(smti->struct_size, "sequence modifier");
+
+	smd->type = type;
+	smd->flag |= SEQUENCE_MODIFIER_EXPANDED;
+
+	BLI_strncpy(smd->name, smti->name, sizeof(smd->name));
+
+	BLI_addtail(&seq->modifiers, smd);
+
+	BKE_sequence_modifier_unique_name(seq, smd);
+
+	if (smti->init_data)
+		smti->init_data(smd);
+}
+
+void BKE_sequence_modifier_free(SequenceModifierData *smd)
+{
+	SequenceModifierTypeInfo *smti = BKE_sequence_modifier_type_info_get(smd->type);
+
+	if (smti && smti->free_data) {
+		smti->free_data(smd);
+	}
+
+	MEM_freeN(smd);
+}
+
+void BKE_sequence_modifier_unique_name(Sequence *seq, SequenceModifierData *smd)
+{
+	SequenceModifierTypeInfo *smti = BKE_sequence_modifier_type_info_get(smd->type);
+
+	BLI_uniquename(&seq->modifiers, smd, smti->name, '.', offsetof(SequenceModifierData, name), sizeof(smd->name));
+}
+
+SequenceModifierData *BKE_sequence_modifier_find_by_name(Sequence *seq, char *name)
+{
+	return BLI_findstring(&(seq->modifiers), name, offsetof(SequenceModifierData, name));
+}
+
+ImBuf *BKE_sequence_modifier_apply_stack(SeqRenderData context, Sequence *seq, ImBuf *ibuf, int cfra)
+{
+	SequenceModifierData *smd;
+	ImBuf *processed_ibuf = ibuf;
+
+	for (smd = seq->modifiers.first; smd; smd = smd->next) {
+		SequenceModifierTypeInfo *smti = BKE_sequence_modifier_type_info_get(smd->type);
+		ImBuf *ibuf_new;
+
+		/* could happen if modifier is being removed or not exists in current version of blender */
+		if (!smti)
+			continue;
+
+		/* modifier is muted, do nothing */
+		if (smd->flag & SEQUENCE_MODIFIER_MUTE)
+			continue;
+
+		if (smti->apply) {
+			ImBuf *mask = modifier_mask_get(smd, context, cfra, ibuf->rect_float != NULL);
+
+			if (processed_ibuf == ibuf)
+				processed_ibuf = IMB_dupImBuf(ibuf);
+
+			ibuf_new = smti->apply(smd, processed_ibuf, mask);
+
+			if (ibuf_new != processed_ibuf) {
+				IMB_freeImBuf(processed_ibuf);
+				processed_ibuf = ibuf_new;
+			}
+
+			if (mask)
+				IMB_freeImBuf(mask);
+		}
+	}
+
+	return processed_ibuf;
+}
+
+void BKE_sequence_modifier_list_copy(Sequence *seqn, Sequence *seq)
+{
+	SequenceModifierData *smd;
+
+	for (smd = seq->modifiers.first; smd; smd = smd->next) {
+		SequenceModifierData *smdn;
+		SequenceModifierTypeInfo *smti = BKE_sequence_modifier_type_info_get(smd->type);
+
+		smdn = MEM_dupallocN(smd);
+
+		if (smti && smti->copy_data)
+			smti->copy_data(smdn, smd);
+
+		smdn->next = smdn->prev = NULL;
+		BLI_addtail(&seqn->modifiers, smdn);
+	}
+}
+
+int BKE_sequence_supports_modifiers(Sequence *seq)
+{
+	return !ELEM(seq->type, SEQ_TYPE_SOUND_RAM, SEQ_TYPE_SOUND_HD);
+}