Merge pull request #680 from jdgleaver/libretro-vulkan-fix

(libretro/Vulkan) Fix glslang shader-related crashes

dep/glslang/OGLCompilersDLL/InitializeDll.cpp

@@ -158,6 +158,7 @@ bool DetachProcess()
 
     OS_FreeTLSIndex(ThreadInitializeIndex);
     ThreadInitializeIndex = OS_INVALID_TLS_INDEX;
+    DeinitializePoolIndex();
 
     return success;
 }

dep/glslang/glslang/Include/InitializeGlobals.h

@@ -38,6 +38,7 @@
 namespace glslang {
 
 bool InitializePoolIndex();
+bool DeinitializePoolIndex();
 
 } // end namespace glslang
 

dep/glslang/glslang/MachineIndependent/PoolAlloc.cpp

@@ -65,6 +65,15 @@ bool InitializePoolIndex()
     return true;
 }
 
+bool DeinitializePoolIndex()
+{
+    if (PoolIndex == OS_INVALID_TLS_INDEX)
+        return false;
+    OS_FreeTLSIndex(PoolIndex);
+    PoolIndex = OS_INVALID_TLS_INDEX;
+    return true;
+}
+
 //
 // Implement the functionality of the TPoolAllocator class, which
 // is documented in PoolAlloc.h.

dep/glslang/glslang/MachineIndependent/ShaderLang.cpp

@@ -1429,6 +1429,7 @@ int ShFinalize()
     glslang::HlslScanContext::deleteKeywordMap();
 #endif
 
+    DetachProcess();
     return 1;
 }
 

src/common/vulkan/shader_compiler.cpp

@@ -24,6 +24,8 @@ bool InitializeGlslang();
 
 static unsigned s_next_bad_shader_id = 1;
 
+static bool glslang_initialized = false;
+
 static std::optional<SPIRVCodeVector> CompileShaderToSPV(EShLanguage stage, const char* stage_filename,
                                                          std::string_view source)
 {
@@ -113,7 +115,6 @@ static std::optional<SPIRVCodeVector> CompileShaderToSPV(EShLanguage stage, cons
 
 bool InitializeGlslang()
 {
-  static bool glslang_initialized = false;
   if (glslang_initialized)
     return true;
 
@@ -123,12 +124,23 @@ bool InitializeGlslang()
     return false;
   }
 
+#ifndef LIBRETRO
   std::atexit([]() { glslang::FinalizeProcess(); });
+#endif
 
   glslang_initialized = true;
   return true;
 }
 
+void DeinitializeGlslang()
+{
+  if (!glslang_initialized)
+    return;
+
+  glslang::FinalizeProcess();
+  glslang_initialized = false;
+}
+
 std::optional<SPIRVCodeVector> CompileVertexShader(std::string_view source_code)
 {
   return CompileShaderToSPV(EShLangVertex, "vs", source_code);

src/common/vulkan/shader_compiler.h

@@ -21,6 +21,8 @@ enum class Type
   Compute
 };
 
+void DeinitializeGlslang();
+
 // SPIR-V compiled code type
 using SPIRVCodeType = u32;
 using SPIRVCodeVector = std::vector<SPIRVCodeType>;

src/duckstation-libretro/libretro_vulkan_host_display.cpp

@@ -136,6 +136,7 @@ void LibretroVulkanHostDisplay::DestroyResources()
   VulkanHostDisplay::DestroyResources();
   Vulkan::Util::SafeDestroyFramebuffer(m_frame_framebuffer);
   m_frame_texture.Destroy();
+  Vulkan::ShaderCompiler::DeinitializeGlslang();
 }
 
 VkRenderPass LibretroVulkanHostDisplay::GetRenderPassForDisplay() const