Freestyle: Fix for incorrect comparisons of Nature values with integer and boolean values.

The problem is that comparisons involving the constants Nature.POINT (for vertices) and
Nature.NO_FEATURE (for edges) were evaluated in a wrong way.  It is recalled that the
Nature class is a subclass of Python's built-in int type, and that these two constants are zero
when evaluated as numbers.  The issue was caused by the implementation of the constants
in an incompatible way for comparison with Python int (and boolean) values.  Specifically,
the zero of Python int is represented by an empty array of digits, whereas the zero-valued
Nature constants were represented by an array of size 1.  Python int comparison operators
first check the lengths of the arrays of two operands, and then start comparing the digits
only when the array length is the same.  For this reason, the two Nature constants were
not properly compared with int values (and thus with boolean values).  It is noted that the
zero-valued Nature constants may result from bitwise operations on other Nature constants
(e.g., Nature.SILHOUETTE & Nature.BORDER), so this issue must have affected many
existing style modules.

The problem was reported by Folkert de Vries (flokkievids) through personal communications.
Thanks a lot!

1 files changed, 14 insertions, 16 deletions

diff --git a/source/blender/freestyle/intern/python/BPy_Nature.cpp b/source/blender/freestyle/intern/python/BPy_Nature.cpp
index da09de99590..bf571bada6a 100644
--- a/source/blender/freestyle/intern/python/BPy_Nature.cpp
+++ b/source/blender/freestyle/intern/python/BPy_Nature.cpp
@@ -35,7 +35,6 @@ extern "C" {
 static PyObject *BPy_Nature_and(PyObject *a, PyObject *b);
 static PyObject *BPy_Nature_xor(PyObject *a, PyObject *b);
 static PyObject *BPy_Nature_or(PyObject *a, PyObject *b);
-static int BPy_Nature_bool(PyObject *v);
 
 /*-----------------------BPy_Nature number method definitions --------------------*/
 
@@ -49,7 +48,7 @@ static PyNumberMethods nature_as_number = {
 	0,                              /* unaryfunc nb_negative */
 	0,                              /* unaryfunc nb_positive */
 	0,                              /* unaryfunc nb_absolute */
-	(inquiry)BPy_Nature_bool,       /* inquiry nb_bool */
+	0,                              /* inquiry nb_bool */
 	0,                              /* unaryfunc nb_invert */
 	0,                              /* binaryfunc nb_lshift */
 	0,                              /* binaryfunc nb_rshift */
@@ -151,7 +150,7 @@ PyTypeObject Nature_Type = {
 /*-----------------------BPy_Nature instance definitions ----------------------------------*/
 
 static PyLongObject _Nature_POINT = {
-	PyVarObject_HEAD_INIT(&Nature_Type, 1)
+	PyVarObject_HEAD_INIT(&Nature_Type, 0)
 	{ Nature::POINT }
 };
 static PyLongObject _Nature_S_VERTEX = {
@@ -175,7 +174,7 @@ static PyLongObject _Nature_CUSP = {
 	{ Nature::CUSP }
 };
 static PyLongObject _Nature_NO_FEATURE = {
-	PyVarObject_HEAD_INIT(&Nature_Type, 1)
+	PyVarObject_HEAD_INIT(&Nature_Type, 0)
 	{ Nature::NO_FEATURE }
 };
 static PyLongObject _Nature_SILHOUETTE = {
@@ -263,7 +262,7 @@ int Nature_Init(PyObject *module)
 static PyObject *BPy_Nature_bitwise(PyObject *a, int op, PyObject *b)
 {
 	BPy_Nature *result;
-	long op1, op2;
+	long op1, op2, v;
 
 	if (!BPy_Nature_Check(a) || !BPy_Nature_Check(b)) {
 		PyErr_SetString(PyExc_TypeError, "operands must be a Nature object");
@@ -279,20 +278,24 @@ static PyObject *BPy_Nature_bitwise(PyObject *a, int op, PyObject *b)
 		PyErr_SetString(PyExc_ValueError, "operand 2: unexpected Nature value");
 		return NULL;
 	}
-	result = PyObject_NewVar(BPy_Nature, &Nature_Type, 1);
-	if (!result)
-		return NULL;
 	switch (op) {
 	case '&':
-		result->i.ob_digit[0] = op1 & op2;
+		v = op1 & op2;
 		break;
 	case '^':
-		result->i.ob_digit[0] = op1 ^ op2;
+		v = op1 ^ op2;
 		break;
 	case '|':
-		result->i.ob_digit[0] = op1 | op2;
+		v = op1 | op2;
 		break;
 	}
+	if (v == 0)
+		result = PyObject_NewVar(BPy_Nature, &Nature_Type, 0);
+	else {
+		result = PyObject_NewVar(BPy_Nature, &Nature_Type, 1);
+		if (result)
+			result->i.ob_digit[0] = v;
+	}
 	return (PyObject *)result;
 }
 
@@ -311,11 +314,6 @@ static PyObject *BPy_Nature_or(PyObject *a, PyObject *b)
 	return BPy_Nature_bitwise(a, '|', b);
 }
 
-static int BPy_Nature_bool(PyObject *v)
-{
-	return ((PyLongObject *)v)->ob_digit[0] != 0;
-}
-
 ///////////////////////////////////////////////////////////////////////////////////////////
 
 #ifdef __cplusplus