Qt: Work around QtWayland bugs

- Render to main no longer screws up the game list/menu bar.
- Toggling render to main no longer breaks the main window.

Positioning still sucks, but the various groups involved would
rather sit around arguing with each other rather than actually
shipping solutions ¯\_(ツ)_/¯.

src/duckstation-qt/displaywidget.cpp

@@ -37,6 +37,7 @@ DisplayWidget::DisplayWidget(QWidget* parent) : QWidget(parent)
   // We want a native window for both D3D and OpenGL.
   setAutoFillBackground(false);
   setAttribute(Qt::WA_NativeWindow, true);
+  setAttribute(Qt::WA_DontCreateNativeAncestors, true);
   setAttribute(Qt::WA_NoSystemBackground, true);
   setAttribute(Qt::WA_PaintOnScreen, true);
   setAttribute(Qt::WA_KeyCompression, false);
@@ -404,7 +405,7 @@ bool DisplayContainer::isNeeded(bool fullscreen, bool render_to_main)
 #if defined(_WIN32) || defined(__APPLE__)
   return false;
 #else
-  if (!isRunningOnWayland())
+  if (!QtHost::IsRunningOnWayland())
     return false;
 
   // We only need this on Wayland because of client-side decorations...
@@ -412,16 +413,6 @@ bool DisplayContainer::isNeeded(bool fullscreen, bool render_to_main)
 #endif
 }
 
-bool DisplayContainer::isRunningOnWayland()
-{
-#if defined(_WIN32) || defined(__APPLE__)
-  return false;
-#else
-  const QString platform_name = QGuiApplication::platformName();
-  return (platform_name == QStringLiteral("wayland"));
-#endif
-}
-
 void DisplayContainer::setDisplayWidget(DisplayWidget* widget)
 {
   Assert(!m_display_widget);
@@ -473,6 +464,7 @@ AuxiliaryDisplayWidget::AuxiliaryDisplayWidget(QWidget* parent, u32 width, u32 h
   // We want a native window for both D3D and OpenGL.
   setAutoFillBackground(false);
   setAttribute(Qt::WA_NativeWindow, true);
+  setAttribute(Qt::WA_DontCreateNativeAncestors, true);
   setAttribute(Qt::WA_NoSystemBackground, true);
   setAttribute(Qt::WA_PaintOnScreen, true);
   setAttribute(Qt::WA_KeyCompression, false);

src/duckstation-qt/displaywidget.h

@@ -77,9 +77,6 @@ public:
   DisplayContainer();
   ~DisplayContainer();
 
-  // Wayland is broken in lots of ways, so we need to check for it.
-  static bool isRunningOnWayland();
-
   static bool isNeeded(bool fullscreen, bool render_to_main);
 
   void setDisplayWidget(DisplayWidget* widget);

src/duckstation-qt/mainwindow.cpp

@@ -101,16 +101,6 @@ static constexpr char DISC_IMAGE_FILTER[] = QT_TRANSLATE_NOOP(
 
 MainWindow* g_main_window = nullptr;
 
-#if defined(_WIN32) || defined(__APPLE__)
-static const bool s_use_central_widget = false;
-#else
-// Qt Wayland is broken. Any sort of stacked widget usage fails to update,
-// leading to broken window resizes, no display rendering, etc. So, we mess
-// with the central widget instead. Which we can't do on xorg, because it
-// breaks window resizing there...
-static bool s_use_central_widget = false;
-#endif
-
 // UI thread VM validity.
 static bool s_disable_window_rounded_corners = false;
 static bool s_system_valid = false;
@@ -151,11 +141,6 @@ MainWindow::MainWindow() : QMainWindow(nullptr)
 {
   Assert(!g_main_window);
   g_main_window = this;
-
-#if !defined(_WIN32) && !defined(__APPLE__)
-  s_use_central_widget = DisplayContainer::isRunningOnWayland();
-#endif
-
   initialize();
 }
 
@@ -260,12 +245,14 @@ bool MainWindow::nativeEvent(const QByteArray& eventType, void* message, qintptr
 
 #endif
 
-std::optional<WindowInfo> MainWindow::acquireRenderWindow(RenderAPI render_api, bool fullscreen, bool render_to_main,
+std::optional<WindowInfo> MainWindow::acquireRenderWindow(RenderAPI render_api, bool fullscreen,
+                                                          bool exclusive_fullscreen, bool render_to_main,
                                                           bool surfaceless, bool use_main_window_pos, Error* error)
 {
-  DEV_LOG("acquireRenderWindow() fullscreen={} render_to_main={} surfaceless={} use_main_window_pos={}",
-          fullscreen ? "true" : "false", render_to_main ? "true" : "false", surfaceless ? "true" : "false",
-          use_main_window_pos ? "true" : "false");
+  DEV_LOG("acquireRenderWindow() fullscreen={} exclusive_fullscreen={}, render_to_main={} surfaceless={} "
+          "use_main_window_pos={}",
+          fullscreen ? "true" : "false", exclusive_fullscreen ? "true" : "false", render_to_main ? "true" : "false",
+          surfaceless ? "true" : "false", use_main_window_pos ? "true" : "false");
 
   QWidget* container =
     m_display_container ? static_cast<QWidget*>(m_display_container) : static_cast<QWidget*>(m_display_widget);
@@ -273,6 +260,9 @@ std::optional<WindowInfo> MainWindow::acquireRenderWindow(RenderAPI render_api,
   const bool is_rendering_to_main = isRenderingToMain();
   const bool changing_surfaceless = (!m_display_widget != surfaceless);
 
+  // Always update exclusive fullscreen state, it controls main window visibility
+  m_exclusive_fullscreen_requested = !surfaceless && exclusive_fullscreen;
+
   // Skip recreating the surface if we're just transitioning between fullscreen and windowed with render-to-main off.
   // .. except on Wayland, where everything tends to break if you don't recreate.
   const bool has_container = (m_display_container != nullptr);
@@ -281,6 +271,7 @@ std::optional<WindowInfo> MainWindow::acquireRenderWindow(RenderAPI render_api,
       !needs_container && !changing_surfaceless)
   {
     DEV_LOG("Toggling to {} without recreating surface", (fullscreen ? "fullscreen" : "windowed"));
+    m_exclusive_fullscreen_requested = exclusive_fullscreen;
 
     // since we don't destroy the display widget, we need to save it here
     if (!is_fullscreen && !is_rendering_to_main)
@@ -357,7 +348,7 @@ void MainWindow::createDisplayWidget(bool fullscreen, bool render_to_main, bool
   }
   else
   {
-    m_display_widget = new DisplayWidget((!fullscreen && render_to_main) ? getContentParent() : nullptr);
+    m_display_widget = new DisplayWidget((!fullscreen && render_to_main) ? m_ui.mainContainer : nullptr);
     container = m_display_widget;
   }
 
@@ -369,22 +360,16 @@ void MainWindow::createDisplayWidget(bool fullscreen, bool render_to_main, bool
 
   if (fullscreen)
   {
-    // Don't risk doing this on Wayland, it really doesn't like window state changes,
-    // and positioning has no effect anyway.
-    if (!s_use_central_widget)
-    {
-      if (isVisible() && g_emu_thread->shouldRenderToMain())
-        container->move(pos());
-      else
-        restoreDisplayWindowGeometryFromConfig();
-    }
+    if (isVisible() && QtHost::CanRenderToMainWindow())
+      container->move(pos());
+    else
+      restoreDisplayWindowGeometryFromConfig();
 
     container->showFullScreen();
   }
   else if (!render_to_main)
   {
-    // See lameland comment above.
-    if (use_main_window_pos && !s_use_central_widget)
+    if (use_main_window_pos)
       container->setGeometry(geometry());
     else
       restoreDisplayWindowGeometryFromConfig();
@@ -393,15 +378,6 @@ void MainWindow::createDisplayWidget(bool fullscreen, bool render_to_main, bool
     if (s_disable_window_rounded_corners)
       PlatformMisc::SetWindowRoundedCornerState(reinterpret_cast<void*>(container->winId()), false);
   }
-  else if (s_use_central_widget)
-  {
-    m_game_list_widget->setVisible(false);
-    takeCentralWidget();
-    m_game_list_widget->setParent(this); // takeCentralWidget() removes parent
-    setCentralWidget(m_display_widget);
-    m_display_widget->setFocus();
-    update();
-  }
   else
   {
     AssertMsg(m_ui.mainContainer->count() == 1, "Has no display widget");
@@ -443,6 +419,7 @@ void MainWindow::releaseRenderWindow()
   // Now we can safely destroy the display window.
   destroyDisplayWidget(true);
   m_display_created = false;
+  m_exclusive_fullscreen_requested = false;
 
   updateDisplayRelatedActions(false, false, false);
   updateShortcutActions(false);
@@ -464,26 +441,12 @@ void MainWindow::destroyDisplayWidget(bool show_game_list)
 
   if (isRenderingToMain())
   {
-    if (s_use_central_widget)
-    {
-      AssertMsg(centralWidget() == m_display_widget, "Display widget is currently central");
-      takeCentralWidget();
-      if (show_game_list)
-      {
-        m_game_list_widget->setVisible(true);
-        setCentralWidget(m_game_list_widget);
-        m_game_list_widget->resizeTableViewColumnsToFit();
-      }
-    }
-    else
+    AssertMsg(m_ui.mainContainer->indexOf(m_display_widget) == 1, "Display widget in stack");
+    m_ui.mainContainer->removeWidget(m_display_widget);
+    if (show_game_list)
     {
-      AssertMsg(m_ui.mainContainer->indexOf(m_display_widget) == 1, "Display widget in stack");
-      m_ui.mainContainer->removeWidget(m_display_widget);
-      if (show_game_list)
-      {
-        m_ui.mainContainer->setCurrentIndex(0);
-        m_game_list_widget->resizeTableViewColumnsToFit();
-      }
+      m_ui.mainContainer->setCurrentIndex(0);
+      m_game_list_widget->resizeTableViewColumnsToFit();
     }
   }
 
@@ -527,11 +490,6 @@ void MainWindow::focusDisplayWidget()
   m_display_widget->setFocus();
 }
 
-QWidget* MainWindow::getContentParent()
-{
-  return s_use_central_widget ? static_cast<QWidget*>(this) : static_cast<QWidget*>(m_ui.mainContainer);
-}
-
 QWidget* MainWindow::getDisplayContainer() const
 {
   return (m_display_container ? static_cast<QWidget*>(m_display_container) : static_cast<QWidget*>(m_display_widget));
@@ -1667,17 +1625,9 @@ void MainWindow::setupAdditionalUi()
   m_ui.actionViewLockToolbar->setChecked(toolbars_locked);
   m_ui.toolBar->setMovable(!toolbars_locked);
 
-  m_game_list_widget = new GameListWidget(getContentParent());
+  m_game_list_widget = new GameListWidget(m_ui.mainContainer);
   m_game_list_widget->initialize();
-  if (s_use_central_widget)
-  {
-    m_ui.mainContainer = nullptr; // setCentralWidget() will delete this
-    setCentralWidget(m_game_list_widget);
-  }
-  else
-  {
-    m_ui.mainContainer->addWidget(m_game_list_widget);
-  }
+  m_ui.mainContainer->addWidget(m_game_list_widget);
 
   m_status_progress_widget = new QProgressBar(m_ui.statusBar);
   m_status_progress_widget->setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Fixed);
@@ -2014,26 +1964,20 @@ void MainWindow::clearProgressBar()
 
 bool MainWindow::isShowingGameList() const
 {
-  if (s_use_central_widget)
-    return (centralWidget() == m_game_list_widget);
-  else
-    return (m_ui.mainContainer->currentIndex() == 0);
+  return (m_ui.mainContainer->currentIndex() == 0);
 }
 
 bool MainWindow::isRenderingFullscreen() const
 {
-  if (!g_gpu_device || !m_display_widget)
+  if (!m_display_widget)
     return false;
 
-  return getDisplayContainer()->isFullScreen();
+  return (m_exclusive_fullscreen_requested || getDisplayContainer()->isFullScreen());
 }
 
 bool MainWindow::isRenderingToMain() const
 {
-  if (s_use_central_widget)
-    return (m_display_widget && centralWidget() == m_display_widget);
-  else
-    return (m_display_widget && m_ui.mainContainer->indexOf(m_display_widget) == 1);
+  return (m_display_widget && m_ui.mainContainer->indexOf(m_display_widget) == 1);
 }
 
 bool MainWindow::shouldHideMouseCursor() const
@@ -2045,7 +1989,7 @@ bool MainWindow::shouldHideMouseCursor() const
 bool MainWindow::shouldHideMainWindow() const
 {
   return Host::GetBoolSettingValue("Main", "HideMainWindowWhenRunning", false) ||
-         (g_emu_thread->shouldRenderToMain() && !isRenderingToMain()) || QtHost::InNoGUIMode();
+         (QtHost::CanRenderToMainWindow() && isRenderingFullscreen()) || QtHost::InNoGUIMode();
 }
 
 void MainWindow::switchToGameListView()
@@ -3030,7 +2974,7 @@ MainWindow::SystemLock MainWindow::pauseAndLockSystem()
   // On MacOS, it forces a workspace switch, which is kinda jarring.
 
 #ifndef __APPLE__
-  const bool was_fullscreen = g_emu_thread->isFullscreen() && !s_use_central_widget;
+  const bool was_fullscreen = g_emu_thread->isFullscreen();
 #else
   const bool was_fullscreen = false;
 #endif

src/duckstation-qt/mainwindow.h

@@ -132,8 +132,9 @@ private Q_SLOTS:
   bool confirmMessage(const QString& title, const QString& message);
   void onStatusMessage(const QString& message);
 
-  std::optional<WindowInfo> acquireRenderWindow(RenderAPI render_api, bool fullscreen, bool render_to_main,
-                                                bool surfaceless, bool use_main_window_pos, Error* error);
+  std::optional<WindowInfo> acquireRenderWindow(RenderAPI render_api, bool fullscreen, bool exclusive_fullscreen,
+                                                bool render_to_main, bool surfaceless, bool use_main_window_pos,
+                                                Error* error);
   void displayResizeRequested(qint32 width, qint32 height);
   void releaseRenderWindow();
   void focusDisplayWidget();
@@ -242,7 +243,6 @@ private:
   void setProgressBar(int current, int total);
   void clearProgressBar();
 
-  QWidget* getContentParent();
   QWidget* getDisplayContainer() const;
   bool isShowingGameList() const;
   bool isRenderingFullscreen() const;
@@ -334,6 +334,7 @@ private:
   bool m_hide_mouse_cursor = false;
 
   bool m_display_created = false;
+  bool m_exclusive_fullscreen_requested = false;
   bool m_save_states_invalidated = false;
   bool m_was_paused_on_surface_loss = false;
   bool m_was_disc_change_request = false;

src/duckstation-qt/qthost.cpp

@@ -1,4 +1,4 @@
-// SPDX-FileCopyrightText: 2019-2024 Connor McLaughlin <stenzek@gmail.com>
+// SPDX-FileCopyrightText: 2019-2025 Connor McLaughlin <stenzek@gmail.com>
 // SPDX-License-Identifier: CC-BY-NC-ND-4.0
 
 #include "qthost.h"
@@ -173,6 +173,15 @@ bool QtHost::PerformEarlyHardwareChecks()
 
 bool QtHost::EarlyProcessStartup()
 {
+#if !defined(_WIN32) && !defined(__APPLE__)
+  // On Wayland, turning any window into a native window causes DPI scaling to break, as well as window
+  // updates, creating a complete mess of a window. Setting this attribute isn't ideal, since you'd think
+  // that setting WA_DontCreateNativeAncestors on the widget would be sufficient, but apparently not.
+  // TODO: Re-evaluate this on Qt 6.9.
+  if (QtHost::IsRunningOnWayland())
+    QGuiApplication::setAttribute(Qt::AA_DontCreateNativeWidgetSiblings, true);
+#endif
+
   // Config-based RAIntegration switch must happen before the main window is displayed.
 #ifdef ENABLE_RAINTEGRATION
   if (!Achievements::IsUsingRAIntegration() && Host::GetBaseBoolSettingValue("Cheevos", "UseRAIntegration", false))
@@ -198,6 +207,16 @@ bool QtHost::InNoGUIMode()
   return s_nogui_mode;
 }
 
+bool QtHost::IsRunningOnWayland()
+{
+#if defined(_WIN32) || defined(__APPLE__)
+  return false;
+#else
+  const QString platform_name = QGuiApplication::platformName();
+  return (platform_name == QStringLiteral("wayland"));
+#endif
+}
+
 QString QtHost::GetAppNameAndVersion()
 {
   return QStringLiteral("DuckStation %1").arg(QLatin1StringView(g_scm_tag_str));
@@ -591,7 +610,7 @@ void EmuThread::checkForSettingsChanges(const Settings& old_settings)
   // don't mess with fullscreen while locked
   if (!QtHost::IsSystemLocked())
   {
-    const bool render_to_main = shouldRenderToMain();
+    const bool render_to_main = QtHost::CanRenderToMainWindow();
     if (m_is_rendering_to_main != render_to_main && !m_is_fullscreen)
     {
       m_is_rendering_to_main = render_to_main;
@@ -656,9 +675,9 @@ void QtHost::MigrateSettings()
   }
 }
 
-bool EmuThread::shouldRenderToMain() const
+bool QtHost::CanRenderToMainWindow()
 {
-  return !Host::GetBoolSettingValue("Main", "RenderToSeparateWindow", false) && !QtHost::InNoGUIMode();
+  return !Host::GetBoolSettingValue("Main", "RenderToSeparateWindow", false) && !InNoGUIMode();
 }
 
 void Host::RequestResizeHostDisplay(s32 new_window_width, s32 new_window_height)
@@ -741,7 +760,7 @@ void EmuThread::startFullscreenUI()
   // we want settings loaded so we choose the correct renderer
   // this also sorts out input sources.
   System::LoadSettings(false);
-  m_is_rendering_to_main = shouldRenderToMain();
+  m_is_rendering_to_main = QtHost::CanRenderToMainWindow();
 
   // borrow the game start fullscreen flag
   const bool start_fullscreen =
@@ -796,7 +815,7 @@ void EmuThread::bootSystem(std::shared_ptr<SystemBootParameters> params)
   if (System::IsValidOrInitializing())
     return;
 
-  m_is_rendering_to_main = shouldRenderToMain();
+  m_is_rendering_to_main = QtHost::CanRenderToMainWindow();
 
   Error error;
   if (!System::BootSystem(std::move(*params), &error))
@@ -928,7 +947,7 @@ void EmuThread::setFullscreen(bool fullscreen, bool allow_render_to_main)
     return;
 
   m_is_fullscreen = fullscreen;
-  m_is_rendering_to_main = allow_render_to_main && shouldRenderToMain();
+  m_is_rendering_to_main = allow_render_to_main && QtHost::CanRenderToMainWindow();
   GPUThread::UpdateDisplayWindow(fullscreen);
 }
 
@@ -984,10 +1003,10 @@ std::optional<WindowInfo> EmuThread::acquireRenderWindow(RenderAPI render_api, b
 
   const bool window_fullscreen = m_is_fullscreen && !exclusive_fullscreen;
   const bool render_to_main = !fullscreen && m_is_rendering_to_main;
-  const bool use_main_window_pos = shouldRenderToMain();
+  const bool use_main_window_pos = QtHost::CanRenderToMainWindow();
 
-  return emit onAcquireRenderWindowRequested(render_api, window_fullscreen, render_to_main, m_is_surfaceless,
-                                             use_main_window_pos, error);
+  return emit onAcquireRenderWindowRequested(render_api, window_fullscreen, exclusive_fullscreen, render_to_main,
+                                             m_is_surfaceless, use_main_window_pos, error);
 }
 
 void EmuThread::releaseRenderWindow()

src/duckstation-qt/qthost.h

@@ -110,7 +110,6 @@ public:
   void stopBackgroundControllerPollTimer();
   void wakeThread();
 
-  bool shouldRenderToMain() const;
   void checkForSettingsChanges(const Settings& old_settings);
 
   void bootOrLoadState(std::string path);
@@ -142,7 +141,8 @@ Q_SIGNALS:
   void systemResumed();
   void gameListRefreshed();
   void gameListRowsChanged(const QList<int>& rows_changed);
-  std::optional<WindowInfo> onAcquireRenderWindowRequested(RenderAPI render_api, bool fullscreen, bool render_to_main,
+  std::optional<WindowInfo> onAcquireRenderWindowRequested(RenderAPI render_api, bool fullscreen,
+                                                           bool exclusive_fullscreen, bool render_to_main,
                                                            bool surfaceless, bool use_main_window_pos, Error* error);
   void onResizeRenderWindowRequested(qint32 width, qint32 height);
   void onReleaseRenderWindowRequested();
@@ -329,6 +329,12 @@ bool InBatchMode();
 /// Sets NoGUI mode (implys batch mode, does not display main window, exits on shutdown).
 bool InNoGUIMode();
 
+/// Returns true if the application is running under Wayland.
+bool IsRunningOnWayland();
+
+/// Returns true if rendering to the main window should be allowed.
+bool CanRenderToMainWindow();
+
 /// Executes a function on the UI thread.
 void RunOnUIThread(const std::function<void()>& func, bool block = false);