Fix wrong usages of region align enumerations

`ARegion.alignment` unfortunately is a mixture of value and bitflag
enumerations. When checking for left/right/top/bottom region alignment,
the flags have to be masked out usually.
Most of the fixed cases here probably didn't cause issues in practice,
but could in fact break at any point when surrounding logic changes.

In fact the assert in #region_visible_rect_calc() failed in an older
file from https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=949035. This
fixes it.

3 files changed, 26 insertions, 14 deletions

diff --git a/source/blender/editors/screen/area.c b/source/blender/editors/screen/area.c
index f4d9c353102..39b2c060ba4 100644
--- a/source/blender/editors/screen/area.c
+++ b/source/blender/editors/screen/area.c
@@ -1141,6 +1141,9 @@ static void region_overlap_fix(ScrArea *sa, ARegion *ar)
     }
   }
 
+  /* Guard against flags slipping through that would have to be masked out in usages below. */
+  BLI_assert(align1 == RGN_ALIGN_ENUM_FROM_MASK(align1));
+
   /* translate or close */
   if (ar1) {
     if (align1 == RGN_ALIGN_LEFT) {
@@ -2585,7 +2588,7 @@ void ED_region_panels_draw(const bContext *C, ARegion *ar)
   /* scrollers */
   const rcti *mask = NULL;
   rcti mask_buf;
-  if (ar->runtime.category && (ar->alignment == RGN_ALIGN_RIGHT)) {
+  if (ar->runtime.category && (RGN_ALIGN_ENUM_FROM_MASK(ar->alignment) == RGN_ALIGN_RIGHT)) {
     UI_view2d_mask_from_win(v2d, &mask_buf);
     mask_buf.xmax -= UI_PANEL_CATEGORY_MARGIN_WIDTH;
     mask = &mask_buf;
@@ -3347,7 +3350,9 @@ static void region_visible_rect_calc(ARegion *ar, rcti *rect)
   for (; arn; arn = arn->next) {
     if (ar != arn && arn->overlap) {
       if (BLI_rcti_isect(rect, &arn->winrct, NULL)) {
-        if (ELEM(arn->alignment, RGN_ALIGN_LEFT, RGN_ALIGN_RIGHT)) {
+        int alignment = RGN_ALIGN_ENUM_FROM_MASK(arn->alignment);
+
+        if (ELEM(alignment, RGN_ALIGN_LEFT, RGN_ALIGN_RIGHT)) {
           /* Overlap left, also check 1 pixel offset (2 regions on one side). */
           if (ABS(rect->xmin - arn->winrct.xmin) < 2) {
             rect->xmin = arn->winrct.xmax;
@@ -3358,7 +3363,7 @@ static void region_visible_rect_calc(ARegion *ar, rcti *rect)
             rect->xmax = arn->winrct.xmin;
           }
         }
-        else if (ELEM(arn->alignment, RGN_ALIGN_TOP, RGN_ALIGN_BOTTOM)) {
+        else if (ELEM(alignment, RGN_ALIGN_TOP, RGN_ALIGN_BOTTOM)) {
           /* Same logic as above for vertical regions. */
           if (ABS(rect->ymin - arn->winrct.ymin) < 2) {
             rect->ymin = arn->winrct.ymax;
@@ -3367,7 +3372,7 @@ static void region_visible_rect_calc(ARegion *ar, rcti *rect)
             rect->ymax = arn->winrct.ymin;
           }
         }
-        else if (arn->alignment == RGN_ALIGN_FLOAT) {
+        else if (alignment == RGN_ALIGN_FLOAT) {
           /* Skip floating. */
         }
         else {
diff --git a/source/blender/editors/screen/area_query.c b/source/blender/editors/screen/area_query.c
index 46559efc614..942050aaffd 100644
--- a/source/blender/editors/screen/area_query.c
+++ b/source/blender/editors/screen/area_query.c
@@ -67,10 +67,12 @@ bool ED_region_panel_category_gutter_calc_rect(const ARegion *ar, rcti *r_ar_gut
   if (UI_panel_category_is_visible(ar)) {
     const int category_tabs_width = round_fl_to_int(UI_view2d_scale_get_x(&ar->v2d) *
                                                     UI_PANEL_CATEGORY_MARGIN_WIDTH);
-    if (ar->alignment == RGN_ALIGN_LEFT) {
+    const int alignment = RGN_ALIGN_ENUM_FROM_MASK(ar->alignment);
+
+    if (alignment == RGN_ALIGN_LEFT) {
       r_ar_gutter->xmax = r_ar_gutter->xmin + category_tabs_width;
     }
-    else if (ar->alignment == RGN_ALIGN_RIGHT) {
+    else if (alignment == RGN_ALIGN_RIGHT) {
       r_ar_gutter->xmin = r_ar_gutter->xmax - category_tabs_width;
     }
     else {
@@ -141,14 +143,16 @@ bool ED_region_contains_xy(const ARegion *ar, const int event_xy[2])
       else {
         /* Side-bar & any other kind of overlapping region. */
 
+        const int alignment = RGN_ALIGN_ENUM_FROM_MASK(ar->alignment);
+
         /* Check alignment to avoid region tabs being clipped out
          * by only clipping a single axis for aligned regions. */
-        if (ELEM(ar->alignment, RGN_ALIGN_TOP, RGN_ALIGN_BOTTOM)) {
+        if (ELEM(alignment, RGN_ALIGN_TOP, RGN_ALIGN_BOTTOM)) {
           if (!ED_region_overlap_isect_x_with_margin(ar, event_xy[0], overlap_margin)) {
             return false;
           }
         }
-        else if (ELEM(ar->alignment, RGN_ALIGN_LEFT, RGN_ALIGN_RIGHT)) {
+        else if (ELEM(alignment, RGN_ALIGN_LEFT, RGN_ALIGN_RIGHT)) {
           if (ED_region_panel_category_gutter_isect_xy(ar, event_xy)) {
             /* pass */
           }
diff --git a/source/blender/editors/screen/screen_ops.c b/source/blender/editors/screen/screen_ops.c
index dd09def2df6..9ea8c8293d4 100644
--- a/source/blender/editors/screen/screen_ops.c
+++ b/source/blender/editors/screen/screen_ops.c
@@ -4114,8 +4114,9 @@ void ED_screens_header_tools_menu_create(bContext *C, uiLayout *layout, void *UN
 {
   ScrArea *sa = CTX_wm_area(C);
   ARegion *ar = CTX_wm_region(C);
-  const char *but_flip_str = (ar->alignment == RGN_ALIGN_TOP) ? IFACE_("Flip to Bottom") :
-                                                                IFACE_("Flip to Top");
+  const char *but_flip_str = (RGN_ALIGN_ENUM_FROM_MASK(ar->alignment) == RGN_ALIGN_TOP) ?
+                                 IFACE_("Flip to Bottom") :
+                                 IFACE_("Flip to Top");
   {
     PointerRNA ptr;
     RNA_pointer_create((ID *)CTX_wm_screen(C), &RNA_Space, sa->spacedata.first, &ptr);
@@ -4160,8 +4161,9 @@ void ED_screens_footer_tools_menu_create(bContext *C, uiLayout *layout, void *UN
 {
   ScrArea *sa = CTX_wm_area(C);
   ARegion *ar = CTX_wm_region(C);
-  const char *but_flip_str = (ar->alignment == RGN_ALIGN_TOP) ? IFACE_("Flip to Bottom") :
-                                                                IFACE_("Flip to Top");
+  const char *but_flip_str = (RGN_ALIGN_ENUM_FROM_MASK(ar->alignment) == RGN_ALIGN_TOP) ?
+                                 IFACE_("Flip to Bottom") :
+                                 IFACE_("Flip to Top");
   {
     PointerRNA ptr;
     RNA_pointer_create((ID *)CTX_wm_screen(C), &RNA_Space, sa->spacedata.first, &ptr);
@@ -4186,8 +4188,9 @@ void ED_screens_footer_tools_menu_create(bContext *C, uiLayout *layout, void *UN
 void ED_screens_navigation_bar_tools_menu_create(bContext *C, uiLayout *layout, void *UNUSED(arg))
 {
   const ARegion *ar = CTX_wm_region(C);
-  const char *but_flip_str = (ar->alignment == RGN_ALIGN_LEFT) ? IFACE_("Flip to Right") :
-                                                                 IFACE_("Flip to Left");
+  const char *but_flip_str = (RGN_ALIGN_ENUM_FROM_MASK(ar->alignment) == RGN_ALIGN_LEFT) ?
+                                 IFACE_("Flip to Right") :
+                                 IFACE_("Flip to Left");
 
   /* default is WM_OP_INVOKE_REGION_WIN, which we don't want here. */
   uiLayoutSetOperatorContext(layout, WM_OP_INVOKE_DEFAULT);