add Anti-Aliasing (very rough draft algorithm, NOT FINAL version) to raskter lib. Code is still quite messy but will be replaced when final algo comes in anyway.

10 files changed, 29 insertions, 8 deletions

diff --git a/source/blender/blenkernel/BKE_mask.h b/source/blender/blenkernel/BKE_mask.h
index dd2c7cb3a18..0e93869a8b0 100644
--- a/source/blender/blenkernel/BKE_mask.h
+++ b/source/blender/blenkernel/BKE_mask.h
@@ -170,7 +170,7 @@ void BKE_mask_layer_shape_changed_remove(struct MaskLayer *masklay, int index, i
 /* rasterization */
 int BKE_mask_get_duration(struct Mask *mask);
 void BKE_mask_rasterize(struct Mask *mask, int width, int height, float *buffer,
-                        const short do_aspect_correct);
+                        const short do_aspect_correct, int do_mask_aa);
 
 #define MASKPOINT_ISSEL_ANY(p)          ( ((p)->bezt.f1 | (p)->bezt.f2 | (p)->bezt.f2) & SELECT)
 #define MASKPOINT_ISSEL_KNOT(p)         ( (p)->bezt.f2 & SELECT)
diff --git a/source/blender/blenkernel/intern/mask.c b/source/blender/blenkernel/intern/mask.c
index 012ce97b4d7..7e9f189ae01 100644
--- a/source/blender/blenkernel/intern/mask.c
+++ b/source/blender/blenkernel/intern/mask.c
@@ -2089,7 +2089,7 @@ int BKE_mask_get_duration(Mask *mask)
 
 /* rasterization */
 void BKE_mask_rasterize(Mask *mask, int width, int height, float *buffer,
-                        const short do_aspect_correct)
+                        const short do_aspect_correct, int do_mask_aa)
 {
 	MaskLayer *masklay;
 
@@ -2154,7 +2154,7 @@ void BKE_mask_rasterize(Mask *mask, int width, int height, float *buffer,
 
 				if (tot_diff_point) {
 					PLX_raskterize(diff_points, tot_diff_point,
-					               buffer_tmp, width, height);
+					               buffer_tmp, width, height, do_mask_aa);
 
 					if (tot_diff_feather_points) {
 						PLX_raskterize_feather(diff_points, tot_diff_point,
diff --git a/source/blender/blenkernel/intern/sequencer.c b/source/blender/blenkernel/intern/sequencer.c
index 889eee25d6b..ddf30ecfa81 100644
--- a/source/blender/blenkernel/intern/sequencer.c
+++ b/source/blender/blenkernel/intern/sequencer.c
@@ -2081,7 +2081,8 @@ static ImBuf *seq_render_mask_strip(
 		BKE_mask_rasterize(seq->mask,
 		                   context.rectx, context.recty,
 		                   maskbuf,
-		                   TRUE);
+		                   TRUE,
+				   FALSE /*XXX- TODO: make on/off for anti-aliasing*/);
 
 		fp_src = maskbuf;
 		fp_dst = ibuf->rect_float;
@@ -2104,7 +2105,8 @@ static ImBuf *seq_render_mask_strip(
 		BKE_mask_rasterize(seq->mask,
 		                   context.rectx, context.recty,
 		                   maskbuf,
-		                   TRUE);
+		                   TRUE,
+				   FALSE /*XXX- TODO: mask on/off for anti-aliasing*/);
 
 		fp_src = maskbuf;
 		ub_dst = (unsigned char *)ibuf->rect;
diff --git a/source/blender/blenkernel/intern/tracking.c b/source/blender/blenkernel/intern/tracking.c
index b61a39a2cd2..0aa0d36c537 100644
--- a/source/blender/blenkernel/intern/tracking.c
+++ b/source/blender/blenkernel/intern/tracking.c
@@ -1428,7 +1428,7 @@ static void track_mask_gpencil_layer_rasterize(int frame_width, int frame_height
 					fp[1] = (stroke_points[i].y - marker->search_min[1]) * frame_height / mask_height;
 				}
 
-				PLX_raskterize((float (*)[2])mask_points, stroke->totpoints, mask, mask_width, mask_height);
+				PLX_raskterize((float (*)[2])mask_points, stroke->totpoints, mask, mask_width, mask_height, FALSE /* XXX- TODO: make on/off for AA*/);
 
 				MEM_freeN(mask_points);
 			}
diff --git a/source/blender/compositor/nodes/COM_MaskNode.cpp b/source/blender/compositor/nodes/COM_MaskNode.cpp
index e26756cfc5b..4371a848a3d 100644
--- a/source/blender/compositor/nodes/COM_MaskNode.cpp
+++ b/source/blender/compositor/nodes/COM_MaskNode.cpp
@@ -54,6 +54,7 @@ void MaskNode::convertToOperations(ExecutionSystem *graph, CompositorContext * c
 
 	operation->setMask(mask);
 	operation->setFramenumber(context->getFramenumber());
+    operation->setSmooth((bool)editorNode->custom1);
 
 	graph->addOperation(operation);
 }
diff --git a/source/blender/compositor/operations/COM_MaskOperation.cpp b/source/blender/compositor/operations/COM_MaskOperation.cpp
index bfbf2b42e82..8f7115659a1 100644
--- a/source/blender/compositor/operations/COM_MaskOperation.cpp
+++ b/source/blender/compositor/operations/COM_MaskOperation.cpp
@@ -32,6 +32,7 @@
 
 extern "C" {
 	#include "BKE_mask.h"
+    #include "../../../../intern/raskter/raskter.h"
 }
 
 MaskOperation::MaskOperation(): NodeOperation()
@@ -74,7 +75,10 @@ void *MaskOperation::initializeTileData(rcti *rect, MemoryBuffer **memoryBuffers
 		float *buffer;
 
 		buffer = (float *)MEM_callocN(sizeof(float) * width * height, "rasterized mask");
-		BKE_mask_rasterize(mask, width, height, buffer, TRUE);
+		BKE_mask_rasterize(mask, width, height, buffer, TRUE, this->smooth);
+        if(this->smooth) {
+            PLX_antialias_buffer(buffer, width, height);
+        }
 
 		this->rasterizedMask = buffer;
 	}
diff --git a/source/blender/compositor/operations/COM_MaskOperation.h b/source/blender/compositor/operations/COM_MaskOperation.h
index 9f2c7f53f56..8534cb20416 100644
--- a/source/blender/compositor/operations/COM_MaskOperation.h
+++ b/source/blender/compositor/operations/COM_MaskOperation.h
@@ -40,6 +40,7 @@ protected:
 	int maskWidth;
 	int maskHeight;
 	int framenumber;
+    bool smooth;
 	float *rasterizedMask;
 
 	/**
@@ -59,6 +60,7 @@ public:
 	void setMaskWidth(int width) {this->maskWidth = width;}
 	void setMaskHeight(int height) {this->maskHeight = height;}
 	void setFramenumber(int framenumber) {this->framenumber = framenumber;}
+    void setSmooth(bool smooth) {this->smooth = smooth;}
 
 	void executePixel(float *color, int x, int y, MemoryBuffer *inputBuffers[], void *data);
 };
diff --git a/source/blender/editors/space_node/drawnode.c b/source/blender/editors/space_node/drawnode.c
index 12c369874fe..e6549034a85 100644
--- a/source/blender/editors/space_node/drawnode.c
+++ b/source/blender/editors/space_node/drawnode.c
@@ -2421,6 +2421,8 @@ static void node_composit_buts_viewer_but(uiLayout *layout, bContext *UNUSED(C),
 static void node_composit_buts_mask(uiLayout *layout, bContext *C, PointerRNA *ptr)
 {
 	uiTemplateID(layout, C, ptr, "mask", NULL, NULL, NULL);
+    uiItemR(layout, ptr, "smooth_mask", 0, NULL, ICON_NONE);
+
 }
 
 /* only once called */
diff --git a/source/blender/makesrna/intern/rna_nodetree.c b/source/blender/makesrna/intern/rna_nodetree.c
index 92de882d48b..73e2b5da4f0 100644
--- a/source/blender/makesrna/intern/rna_nodetree.c
+++ b/source/blender/makesrna/intern/rna_nodetree.c
@@ -3089,6 +3089,11 @@ static void def_cmp_mask(StructRNA *srna)
 {
 	PropertyRNA *prop;
 
+    prop = RNA_def_property(srna, "smooth_mask", PROP_BOOLEAN, PROP_NONE);
+    RNA_def_property_boolean_sdna(prop, NULL, "custom1", 0);
+    RNA_def_property_ui_text(prop, "Anti-Alias", "Apply an anti-aliasing filter to the mask");
+    RNA_def_property_update(prop, NC_NODE | NA_EDITED, "rna_Node_update");
+
 	prop = RNA_def_property(srna, "mask", PROP_POINTER, PROP_NONE);
 	RNA_def_property_pointer_sdna(prop, NULL, "id");
 	RNA_def_property_struct_type(prop, "Mask");
diff --git a/source/blender/nodes/composite/nodes/node_composite_mask.c b/source/blender/nodes/composite/nodes/node_composite_mask.c
index dbe3c11ab55..d323839e690 100644
--- a/source/blender/nodes/composite/nodes/node_composite_mask.c
+++ b/source/blender/nodes/composite/nodes/node_composite_mask.c
@@ -38,6 +38,8 @@
 
 #include "node_composite_util.h"
 
+#include "../../../../intern/raskter/raskter.h"
+
 /* **************** Translate  ******************** */
 
 static bNodeSocketTemplate cmp_node_mask_out[] = {
@@ -68,8 +70,11 @@ static void exec(void *data, bNode *node, bNodeStack **UNUSED(in), bNodeStack **
 		stackbuf = alloc_compbuf(sx, sy, CB_VAL, TRUE);
 		res = stackbuf->rect;
 
-		BKE_mask_rasterize(mask, sx, sy, res, TRUE);
+		BKE_mask_rasterize(mask, sx, sy, res, TRUE, node->custom1);
 
+		if(node->custom1){
+			PLX_antialias_buffer(res,sx,sy);
+		}
 		/* pass on output and free */
 		out[0]->data = stackbuf;
 	}