Fix T97731: Python traceback no longer includes line-numbers

Regression caused by [0] that caused the error message to be
created based on a normalized exception (which hid line numbers).

PyC_ExceptionBuffer{_Simple} & BPy_errors_to_report
no longer clears the exception.

This could have been resolved by changing python_script_error_jump
however that would involve changes to reference counting that are more
risky (noted in code-comment).

[0]: 2d2baeaf04d481f284bc2f098fb6d7ee9268151f

7 files changed, 26 insertions, 16 deletions

diff --git a/source/blender/freestyle/intern/system/PythonInterpreter.h b/source/blender/freestyle/intern/system/PythonInterpreter.h
index 8b2d3c5bb60..0c836d447e5 100644
--- a/source/blender/freestyle/intern/system/PythonInterpreter.h
+++ b/source/blender/freestyle/intern/system/PythonInterpreter.h
@@ -90,6 +90,7 @@ class PythonInterpreter : public Interpreter {
 
     if (!BPY_run_string_eval(_context, NULL, str.c_str())) {
       BPy_errors_to_report(reports);
+      PyErr_Clear();
       cerr << "\nError executing Python script from PythonInterpreter::interpretString" << endl;
       cerr << "Name: " << name << endl;
       cerr << "Errors: " << endl;
diff --git a/source/blender/python/generic/py_capi_utils.c b/source/blender/python/generic/py_capi_utils.c
index 3b64fb23460..d2e3c44c1b6 100644
--- a/source/blender/python/generic/py_capi_utils.c
+++ b/source/blender/python/generic/py_capi_utils.c
@@ -892,6 +892,10 @@ PyObject *PyC_ExceptionBuffer(void)
   PySys_SetObject("stderr", string_io);
 
   PyErr_Restore(error_type, error_value, error_traceback);
+  /* Printing clears (call #PyErr_Clear as well to ensure it's cleared).  */
+  Py_XINCREF(error_type);
+  Py_XINCREF(error_value);
+  Py_XINCREF(error_traceback);
   PyErr_Print(); /* print the error */
   PyErr_Clear();
 
@@ -907,17 +911,18 @@ PyObject *PyC_ExceptionBuffer(void)
   Py_DECREF(string_io_getvalue);
   Py_DECREF(string_io); /* free the original reference */
 
-  PyErr_Clear();
+  PyErr_Restore(error_type, error_value, error_traceback);
+
   return string_io_buf;
 
 error_cleanup:
-  /* could not import the module so print the error and close */
+  /* Could not import the module so print the error and close. */
   Py_XDECREF(string_io_mod);
   Py_XDECREF(string_io);
 
   PyErr_Restore(error_type, error_value, error_traceback);
   PyErr_Print(); /* print the error */
-  PyErr_Clear();
+  PyErr_Restore(error_type, error_value, error_traceback);
 
   return NULL;
 }
@@ -925,19 +930,15 @@ error_cleanup:
 
 PyObject *PyC_ExceptionBuffer_Simple(void)
 {
-  PyObject *string_io_buf = NULL;
-
-  PyObject *error_type, *error_value, *error_traceback;
-
   if (!PyErr_Occurred()) {
     return NULL;
   }
 
-  PyErr_Fetch(&error_type, &error_value, &error_traceback);
+  PyObject *string_io_buf = NULL;
 
-  if (error_value == NULL) {
-    return NULL;
-  }
+  PyObject *error_type, *error_value, *error_traceback;
+
+  PyErr_Fetch(&error_type, &error_value, &error_traceback);
 
   if (PyErr_GivenExceptionMatches(error_type, PyExc_SyntaxError)) {
     /* Special exception for syntax errors,
@@ -959,7 +960,6 @@ PyObject *PyC_ExceptionBuffer_Simple(void)
 
   PyErr_Restore(error_type, error_value, error_traceback);
 
-  PyErr_Clear();
   return string_io_buf;
 }
 
diff --git a/source/blender/python/intern/bpy_capi_utils.h b/source/blender/python/intern/bpy_capi_utils.h
index ab5ce7818f3..73a5d2ebc79 100644
--- a/source/blender/python/intern/bpy_capi_utils.h
+++ b/source/blender/python/intern/bpy_capi_utils.h
@@ -38,6 +38,8 @@ bool BPy_errors_to_report_ex(struct ReportList *reports,
  *   BKE_reports_print(reports);
  * }
  * \endcode
+ *
+ * \note The caller is responsible for clearing the error (see #PyErr_Clear).
  */
 bool BPy_errors_to_report(struct ReportList *reports);
 
diff --git a/source/blender/python/intern/bpy_interface_run.c b/source/blender/python/intern/bpy_interface_run.c
index 50a2722c276..886f5ee278d 100644
--- a/source/blender/python/intern/bpy_interface_run.c
+++ b/source/blender/python/intern/bpy_interface_run.c
@@ -128,9 +128,6 @@ static bool python_script_exec(bContext *C,
       Py_DECREF(filepath_dummy_py);
 
       if (PyErr_Occurred()) {
-        if (do_jump) {
-          python_script_error_jump_text(text, filepath_dummy);
-        }
         BPY_text_free_code(text);
       }
     }
@@ -184,6 +181,7 @@ static bool python_script_exec(bContext *C,
   }
 
   if (!py_result) {
+    BPy_errors_to_report(reports);
     if (text) {
       if (do_jump) {
         /* ensure text is valid before use, the script may have freed itself */
@@ -193,7 +191,7 @@ static bool python_script_exec(bContext *C,
         }
       }
     }
-    BPy_errors_to_report(reports);
+    PyErr_Clear();
   }
   else {
     Py_DECREF(py_result);
@@ -275,6 +273,7 @@ static bool bpy_run_string_impl(bContext *C,
     ReportList reports;
     BKE_reports_init(&reports, RPT_STORE);
     BPy_errors_to_report(&reports);
+    PyErr_Clear();
 
     /* Ensure the reports are printed. */
     if (!BKE_reports_print_test(&reports, RPT_ERROR)) {
@@ -336,6 +335,7 @@ static void run_string_handle_error(struct BPy_RunErrInfo *err_info)
   PyObject *py_err_str = err_info->use_single_line_error ? PyC_ExceptionBuffer_Simple() :
                                                            PyC_ExceptionBuffer();
   const char *err_str = py_err_str ? PyUnicode_AsUTF8(py_err_str) : "Unable to extract exception";
+  PyErr_Clear();
 
   if (err_info->reports != NULL) {
     if (err_info->report_prefix) {
diff --git a/source/blender/python/intern/bpy_props.c b/source/blender/python/intern/bpy_props.c
index 893ea90a235..a6aa1f46b0c 100644
--- a/source/blender/python/intern/bpy_props.c
+++ b/source/blender/python/intern/bpy_props.c
@@ -4111,6 +4111,8 @@ StructRNA *pointer_type_from_py(PyObject *value, const char *error_prefix)
     if (PyErr_Occurred()) {
       PyObject *msg = PyC_ExceptionBuffer();
       const char *msg_char = PyUnicode_AsUTF8(msg);
+      PyErr_Clear();
+
       PyErr_Format(
           PyExc_TypeError, "%.200s expected an RNA type, failed with: %s", error_prefix, msg_char);
       Py_DECREF(msg);
diff --git a/source/blender/python/intern/bpy_rna.c b/source/blender/python/intern/bpy_rna.c
index 83e14c305bc..2276e5e97a8 100644
--- a/source/blender/python/intern/bpy_rna.c
+++ b/source/blender/python/intern/bpy_rna.c
@@ -2051,6 +2051,7 @@ static int pyrna_py_to_prop(
                   &itemptr, item, true, "Converting a Python list to an RNA collection") == -1) {
             PyObject *msg = PyC_ExceptionBuffer();
             const char *msg_char = PyUnicode_AsUTF8(msg);
+            PyErr_Clear();
 
             PyErr_Format(PyExc_TypeError,
                          "%.200s %.200s.%.200s error converting a member of a collection "
diff --git a/source/blender/python/intern/bpy_traceback.c b/source/blender/python/intern/bpy_traceback.c
index 1c545774203..45977ba400c 100644
--- a/source/blender/python/intern/bpy_traceback.c
+++ b/source/blender/python/intern/bpy_traceback.c
@@ -165,6 +165,10 @@ finally:
 bool python_script_error_jump(
     const char *filepath, int *r_lineno, int *r_offset, int *r_lineno_end, int *r_offset_end)
 {
+  /* WARNING(@campbellbarton): The normalized exception is restored (loosing line number info).
+   * Ideally this would leave the exception state as it found it, but that needs to be done
+   * carefully with regards to reference counting, see: T97731. */
+
   bool success = false;
   PyObject *exception, *value;
   PyTracebackObject *tb;