Fix crash on exit when background rendering in wayland

Disable libdecor Wayland requirement which would use an X11 fallback.

While the crash could be investigated, using libdecor at all makes
no sense in background mode.

5 files changed, 22 insertions, 8 deletions

diff --git a/intern/ghost/GHOST_ISystem.h b/intern/ghost/GHOST_ISystem.h
index 64b5f4ee496..edaeca1e159 100644
--- a/intern/ghost/GHOST_ISystem.h
+++ b/intern/ghost/GHOST_ISystem.h
@@ -118,9 +118,11 @@ class GHOST_ISystem {
   /**
    * Creates the one and only system.
    * \param verbose: report back-ends that were attempted no back-end could be loaded.
+   * \param background: loading the system for background rendering (no visible windows).
    * \return An indication of success.
    */
-  static GHOST_TSuccess createSystem(bool verbose);
+
+  static GHOST_TSuccess createSystem(bool verbose, bool background);
   static GHOST_TSuccess createSystemBackground();
 
   /**
diff --git a/intern/ghost/intern/GHOST_C-api.cpp b/intern/ghost/intern/GHOST_C-api.cpp
index aa2363c10b3..0c595b27148 100644
--- a/intern/ghost/intern/GHOST_C-api.cpp
+++ b/intern/ghost/intern/GHOST_C-api.cpp
@@ -24,7 +24,7 @@
 
 GHOST_SystemHandle GHOST_CreateSystem(void)
 {
-  GHOST_ISystem::createSystem(true);
+  GHOST_ISystem::createSystem(true, false);
   GHOST_ISystem *system = GHOST_ISystem::getSystem();
 
   return (GHOST_SystemHandle)system;
diff --git a/intern/ghost/intern/GHOST_ISystem.cpp b/intern/ghost/intern/GHOST_ISystem.cpp
index 8e2859ca1e1..696848ce623 100644
--- a/intern/ghost/intern/GHOST_ISystem.cpp
+++ b/intern/ghost/intern/GHOST_ISystem.cpp
@@ -34,7 +34,7 @@ const char *GHOST_ISystem::m_system_backend_id = nullptr;
 
 GHOST_TBacktraceFn GHOST_ISystem::m_backtrace_fn = nullptr;
 
-GHOST_TSuccess GHOST_ISystem::createSystem(bool verbose)
+GHOST_TSuccess GHOST_ISystem::createSystem(bool verbose, [[maybe_unused]] bool background)
 {
   /* When GHOST fails to start, report the back-ends that were attempted.
    * A Verbose argument could be supported in printing isn't always desired. */
@@ -61,7 +61,7 @@ GHOST_TSuccess GHOST_ISystem::createSystem(bool verbose)
     if (has_wayland_libraries) {
       backends_attempted[backends_attempted_num++] = "WAYLAND";
       try {
-        m_system = new GHOST_SystemWayland();
+        m_system = new GHOST_SystemWayland(background);
       }
       catch (const std::runtime_error &) {
         delete m_system;
@@ -99,7 +99,7 @@ GHOST_TSuccess GHOST_ISystem::createSystem(bool verbose)
     if (has_wayland_libraries) {
       backends_attempted[backends_attempted_num++] = "WAYLAND";
       try {
-        m_system = new GHOST_SystemWayland();
+        m_system = new GHOST_SystemWayland(background);
       }
       catch (const std::runtime_error &) {
         delete m_system;
@@ -160,7 +160,7 @@ GHOST_TSuccess GHOST_ISystem::createSystemBackground()
   if (!m_system) {
 #if !defined(WITH_HEADLESS)
     /* Try to create a off-screen render surface with the graphical systems. */
-    success = createSystem(false);
+    success = createSystem(false, true);
     if (success) {
       return success;
     }
diff --git a/intern/ghost/intern/GHOST_SystemWayland.cpp b/intern/ghost/intern/GHOST_SystemWayland.cpp
index 442e51d4f7c..232c17dcde4 100644
--- a/intern/ghost/intern/GHOST_SystemWayland.cpp
+++ b/intern/ghost/intern/GHOST_SystemWayland.cpp
@@ -4162,7 +4162,8 @@ static const struct wl_registry_listener registry_listener = {
  * WAYLAND specific implementation of the #GHOST_System interface.
  * \{ */
 
-GHOST_SystemWayland::GHOST_SystemWayland() : GHOST_System(), display_(new GWL_Display)
+GHOST_SystemWayland::GHOST_SystemWayland(bool background)
+    : GHOST_System(), display_(new GWL_Display)
 {
   wl_log_set_handler_client(ghost_wayland_log_handler);
 
@@ -4187,6 +4188,14 @@ GHOST_SystemWayland::GHOST_SystemWayland() : GHOST_System(), display_(new GWL_Di
   wl_registry_destroy(registry);
 
 #ifdef WITH_GHOST_WAYLAND_LIBDECOR
+  /* Ignore windowing requirements when running in background mode,
+   * as it doesn't make sense to fall back to X11 because of windowing functionality
+   * in background mode, also LIBDECOR is crashing in background mode `blender -b -f 1`
+   * for e.g. while it could be fixed, requiring the library at all makes no sense . */
+  if (background) {
+    display_->libdecor_required = false;
+  }
+
   if (display_->libdecor_required) {
     gwl_xdg_decor_system_destroy(display_->xdg_decor);
     display_->xdg_decor = nullptr;
@@ -4200,6 +4209,8 @@ GHOST_SystemWayland::GHOST_SystemWayland() : GHOST_System(), display_(new GWL_Di
 #  endif
       display_destroy(display_);
       throw std::runtime_error("Wayland: unable to find libdecor!");
+
+      use_libdecor = true;
     }
   }
   else {
diff --git a/intern/ghost/intern/GHOST_SystemWayland.h b/intern/ghost/intern/GHOST_SystemWayland.h
index f08e9fdcf9c..c27f175002e 100644
--- a/intern/ghost/intern/GHOST_SystemWayland.h
+++ b/intern/ghost/intern/GHOST_SystemWayland.h
@@ -84,7 +84,8 @@ struct GWL_Output {
 
 class GHOST_SystemWayland : public GHOST_System {
  public:
-  GHOST_SystemWayland();
+  GHOST_SystemWayland(bool background);
+  GHOST_SystemWayland() : GHOST_SystemWayland(true){};
 
   ~GHOST_SystemWayland() override;