Fix T82002: ENTER does nothing when mouse cursor is over save dialog text field

The `file.execute` operator would early-exit because the mouse wasn't hovering
the file list. Caused by 4ba9d7d71e8f.

Although simpler solutions would have been possible, I decided it's better to
split add a new operator for executing based on the mouse (for double-clicking
files), to separate that from the window level execute operator
(`file.execute`). This allows more control and we can get rid of the implicit
assumption that the keymap would call `file.select` on mouse-press, and
`file.execute` on double-click, for the double-click behavior to work. The cost
is that we execute the file selection & activation logic twice on the
double-click, but that shouldn't be an issue at all.
Also removes the `need_active` property from the `file.execute` operator.
That's ancient and wasn't implemented well anyway.

To be clear, reason this fixes the bug is that `file.execute` works entirely
with the `execute()` callback now and doesn't early-exit based on the mouse
position anymore.

Might trigger warnings about the `need_active` property not being found for
custom keymaps. These can be ignored and the property can safely be removed
from the keymap. I don't expect other keymap breakages.

3 files changed, 78 insertions, 40 deletions

diff --git a/source/blender/editors/space_file/file_intern.h b/source/blender/editors/space_file/file_intern.h
index 309b280177c..f1d0197b9ae 100644
--- a/source/blender/editors/space_file/file_intern.h
+++ b/source/blender/editors/space_file/file_intern.h
@@ -63,6 +63,7 @@ void FILE_OT_bookmark_move(struct wmOperatorType *ot);
 void FILE_OT_reset_recent(wmOperatorType *ot);
 void FILE_OT_hidedot(struct wmOperatorType *ot);
 void FILE_OT_execute(struct wmOperatorType *ot);
+void FILE_OT_mouse_execute(struct wmOperatorType *ot);
 void FILE_OT_cancel(struct wmOperatorType *ot);
 void FILE_OT_parent(struct wmOperatorType *ot);
 void FILE_OT_directory_new(struct wmOperatorType *ot);
diff --git a/source/blender/editors/space_file/file_ops.c b/source/blender/editors/space_file/file_ops.c
index 856bd5b1bc3..61f3c046550 100644
--- a/source/blender/editors/space_file/file_ops.c
+++ b/source/blender/editors/space_file/file_ops.c
@@ -536,6 +536,14 @@ void FILE_OT_select_box(wmOperatorType *ot)
 /** \name Select Pick Operator
  * \{ */
 
+static rcti file_select_mval_to_select_rect(const int mval[2])
+{
+  rcti rect;
+  rect.xmin = rect.xmax = mval[0];
+  rect.ymin = rect.ymax = mval[1];
+  return rect;
+}
+
 static int file_select_invoke(bContext *C, wmOperator *op, const wmEvent *event)
 {
   ARegion *region = CTX_wm_region(C);
@@ -551,8 +559,7 @@ static int file_select_invoke(bContext *C, wmOperator *op, const wmEvent *event)
     return OPERATOR_CANCELLED;
   }
 
-  rect.xmin = rect.xmax = event->mval[0];
-  rect.ymin = rect.ymax = event->mval[1];
+  rect = file_select_mval_to_select_rect(event->mval);
 
   if (!ED_fileselect_layout_is_inside_pt(sfile->layout, &region->v2d, rect.xmin, rect.ymin)) {
     return OPERATOR_CANCELLED | OPERATOR_PASS_THROUGH;
@@ -1711,14 +1718,14 @@ bool file_draw_check_exists(SpaceFile *sfile)
 /** \name Execute File Window Operator
  * \{ */
 
-static int file_exec(bContext *C, wmOperator *exec_op)
+/**
+ * Execute the active file, as set in the file select params.
+ */
+static bool file_execute(bContext *C, SpaceFile *sfile)
 {
   Main *bmain = CTX_data_main(C);
-  wmWindowManager *wm = CTX_wm_manager(C);
-  SpaceFile *sfile = CTX_wm_space_file(C);
   FileSelectParams *params = ED_fileselect_get_active_params(sfile);
-  struct FileDirEntry *file = filelist_file(sfile->files, params->active_file);
-  char filepath[FILE_MAX];
+  FileDirEntry *file = filelist_file(sfile->files, params->active_file);
 
   if (file && file->redirection_path) {
     /* redirection_path is an absolute path that takes precedence
@@ -1753,22 +1760,7 @@ static int file_exec(bContext *C, wmOperator *exec_op)
   /* opening file - sends events now, so things get handled on windowqueue level */
   else if (sfile->op) {
     wmOperator *op = sfile->op;
-
-    /* When used as a macro, for double-click, to prevent closing when double-clicking on item. */
-    if (RNA_boolean_get(exec_op->ptr, "need_active")) {
-      const int numfiles = filelist_files_ensure(sfile->files);
-      int i, active = 0;
-
-      for (i = 0; i < numfiles; i++) {
-        if (filelist_entry_select_index_get(sfile->files, i, CHECK_ALL)) {
-          active = 1;
-          break;
-        }
-      }
-      if (active == 0) {
-        return OPERATOR_CANCELLED;
-      }
-    }
+    char filepath[FILE_MAX];
 
     sfile->op = NULL;
 
@@ -1788,50 +1780,94 @@ static int file_exec(bContext *C, wmOperator *exec_op)
                      BKE_appdir_folder_id_create(BLENDER_USER_CONFIG, NULL),
                      BLENDER_BOOKMARK_FILE);
     fsmenu_write_file(ED_fsmenu_get(), filepath);
-    WM_event_fileselect_event(wm, op, EVT_FILESELECT_EXEC);
+    WM_event_fileselect_event(CTX_wm_manager(C), op, EVT_FILESELECT_EXEC);
   }
 
   return OPERATOR_FINISHED;
 }
 
-static int file_exec_invoke(bContext *C, wmOperator *op, const wmEvent *event)
+static int file_exec(bContext *C, wmOperator *UNUSED(op))
 {
-  ARegion *region = CTX_wm_region(C);
   SpaceFile *sfile = CTX_wm_space_file(C);
 
-  if (!ED_fileselect_layout_is_inside_pt(
-          sfile->layout, &region->v2d, event->mval[0], event->mval[1])) {
-    return OPERATOR_CANCELLED | OPERATOR_PASS_THROUGH;
+  if (!file_execute(C, sfile)) {
+    return OPERATOR_CANCELLED;
   }
 
-  return file_exec(C, op);
+  return OPERATOR_FINISHED;
 }
 
 void FILE_OT_execute(struct wmOperatorType *ot)
 {
-  PropertyRNA *prop;
-
   /* identifiers */
   ot->name = "Execute File Window";
   ot->description = "Execute selected file";
   ot->idname = "FILE_OT_execute";
 
   /* api callbacks */
-  ot->invoke = file_exec_invoke;
   ot->exec = file_exec;
   /* Important since handler is on window level.
    *
    * Avoid using #file_operator_poll since this is also used for entering directories
    * which is used even when the file manager doesn't have an operator. */
   ot->poll = ED_operator_file_active;
+}
 
-  /* properties */
-  prop = RNA_def_boolean(ot->srna,
-                         "need_active",
-                         0,
-                         "Need Active",
-                         "Only execute if there's an active selected file in the file list");
-  RNA_def_property_flag(prop, PROP_SKIP_SAVE);
+/**
+ * \returns false if the mouse doesn't hover a selectable item.
+ */
+static bool file_ensure_hovered_is_active(bContext *C, const wmEvent *event)
+{
+  rcti rect = file_select_mval_to_select_rect(event->mval);
+  if (file_select(C, &rect, FILE_SEL_ADD, false, false) == FILE_SELECT_NOTHING) {
+    return false;
+  }
+
+  return true;
+}
+
+static int file_execute_mouse_invoke(bContext *C, wmOperator *UNUSED(op), const wmEvent *event)
+{
+  ARegion *region = CTX_wm_region(C);
+  SpaceFile *sfile = CTX_wm_space_file(C);
+
+  if (!ED_fileselect_layout_is_inside_pt(
+          sfile->layout, &region->v2d, event->mval[0], event->mval[1])) {
+    return OPERATOR_CANCELLED | OPERATOR_PASS_THROUGH;
+  }
+
+  /* Note that this isn't needed practically, because the keymap already activates the hovered item
+   * on mouse-press. This execute operator is called afterwards on the double-click event then.
+   * However relying on this would be fragile and could break with keymap changes, so better to
+   * have this mouse-execute operator that makes sure once more that the hovered file is active. */
+  if (!file_ensure_hovered_is_active(C, event)) {
+    return OPERATOR_CANCELLED;
+  }
+
+  if (!file_execute(C, sfile)) {
+    return OPERATOR_CANCELLED;
+  }
+
+  return OPERATOR_FINISHED;
+}
+
+/**
+ * Variation of #FILE_OT_execute that accounts for some mouse specific handling. Otherwise calls
+ * the same logic.
+ */
+void FILE_OT_mouse_execute(wmOperatorType *ot)
+{
+  /* identifiers */
+  ot->name = "Execute File";
+  ot->description =
+      "Perform the current execute action for the file under the cursor (e.g. open the file)";
+  ot->idname = "FILE_OT_mouse_execute";
+
+  /* api callbacks */
+  ot->invoke = file_execute_mouse_invoke;
+  ot->poll = ED_operator_file_active;
+
+  ot->flag = OPTYPE_INTERNAL;
 }
 
 /** \} */
diff --git a/source/blender/editors/space_file/space_file.c b/source/blender/editors/space_file/space_file.c
index 4422a685af1..12bc0a68ca6 100644
--- a/source/blender/editors/space_file/space_file.c
+++ b/source/blender/editors/space_file/space_file.c
@@ -663,6 +663,7 @@ static void file_operatortypes(void)
   WM_operatortype_append(FILE_OT_highlight);
   WM_operatortype_append(FILE_OT_sort_column_ui_context);
   WM_operatortype_append(FILE_OT_execute);
+  WM_operatortype_append(FILE_OT_mouse_execute);
   WM_operatortype_append(FILE_OT_cancel);
   WM_operatortype_append(FILE_OT_parent);
   WM_operatortype_append(FILE_OT_previous);