diff --git a/src/video_core/renderer_opengl/gl_rasterizer.cpp b/src/video_core/renderer_opengl/gl_rasterizer.cpp
index ee313cb2f..79d16d1f3 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer.cpp
+++ b/src/video_core/renderer_opengl/gl_rasterizer.cpp
@@ -1022,11 +1022,11 @@ void RasterizerOpenGL::SetupTextures(Maxwell::ShaderStage stage, const Shader& s
         if (surface != nullptr) {
             unit.texture =
                 entry.IsArray() ? surface->TextureLayer().handle : surface->Texture().handle;
-            unit.target = entry.IsArray() ? surface->TargetLayer() : surface->Target();
-            unit.swizzle.r = MaxwellToGL::SwizzleSource(texture.tic.x_source);
-            unit.swizzle.g = MaxwellToGL::SwizzleSource(texture.tic.y_source);
-            unit.swizzle.b = MaxwellToGL::SwizzleSource(texture.tic.z_source);
-            unit.swizzle.a = MaxwellToGL::SwizzleSource(texture.tic.w_source);
+            const GLenum target = entry.IsArray() ? surface->TargetLayer() : surface->Target();
+            surface->UpdateSwizzle(texture.tic.x_source, texture.tic.y_source, texture.tic.z_source,
+                                   texture.tic.w_source);
+            unit.texture = handle;
+            unit.target = target;
         } else {
             // Can occur when texture addr is null or its memory is unmapped/invalid
             unit.texture = 0;
diff --git a/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp b/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp
index a3c782972..0dc4857e8 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp
+++ b/src/video_core/renderer_opengl/gl_rasterizer_cache.cpp
@@ -857,6 +857,8 @@ void CachedSurface::EnsureTextureView() {
     glTextureView(texture_view.handle, target, texture.handle, gl_internal_format, 0,
                   params.max_mip_level, 0, 1);
     ApplyTextureDefaults(texture_view.handle, params.max_mip_level);
+    glTextureParameteriv(texture_view.handle, GL_TEXTURE_SWIZZLE_RGBA,
+                         reinterpret_cast<const GLint*>(swizzle.data()));
 }
 
 MICROPROFILE_DEFINE(OpenGL_TextureUL, "OpenGL", "Texture Upload", MP_RGB(128, 192, 64));
@@ -870,6 +872,24 @@ void CachedSurface::UploadGLTexture(GLuint read_fb_handle, GLuint draw_fb_handle
         UploadGLMipmapTexture(i, read_fb_handle, draw_fb_handle);
 }
 
+void CachedSurface::UpdateSwizzle(Tegra::Texture::SwizzleSource swizzle_x,
+                                  Tegra::Texture::SwizzleSource swizzle_y,
+                                  Tegra::Texture::SwizzleSource swizzle_z,
+                                  Tegra::Texture::SwizzleSource swizzle_w) {
+    const GLenum new_x = MaxwellToGL::SwizzleSource(swizzle_x);
+    const GLenum new_y = MaxwellToGL::SwizzleSource(swizzle_y);
+    const GLenum new_z = MaxwellToGL::SwizzleSource(swizzle_z);
+    const GLenum new_w = MaxwellToGL::SwizzleSource(swizzle_w);
+    if (swizzle[0] != new_x || swizzle[1] != new_y || swizzle[2] != new_z || swizzle[3] != new_w) {
+        swizzle = {new_x, new_y, new_z, new_w};
+        const auto swizzle_data = reinterpret_cast<const GLint*>(swizzle.data());
+        glTextureParameteriv(texture.handle, GL_TEXTURE_SWIZZLE_RGBA, swizzle_data);
+        if (texture_view.handle != 0) {
+            glTextureParameteriv(texture_view.handle, GL_TEXTURE_SWIZZLE_RGBA, swizzle_data);
+        }
+    }
+}
+
 RasterizerCacheOpenGL::RasterizerCacheOpenGL(RasterizerOpenGL& rasterizer)
     : RasterizerCache{rasterizer} {
     read_framebuffer.Create();
diff --git a/src/video_core/renderer_opengl/gl_rasterizer_cache.h b/src/video_core/renderer_opengl/gl_rasterizer_cache.h
index 9ee6f3f65..490b8252e 100644
--- a/src/video_core/renderer_opengl/gl_rasterizer_cache.h
+++ b/src/video_core/renderer_opengl/gl_rasterizer_cache.h
@@ -382,6 +382,11 @@ public:
     // Upload data in gl_buffer to this surface's texture
     void UploadGLTexture(GLuint read_fb_handle, GLuint draw_fb_handle);
 
+    void UpdateSwizzle(Tegra::Texture::SwizzleSource swizzle_x,
+                       Tegra::Texture::SwizzleSource swizzle_y,
+                       Tegra::Texture::SwizzleSource swizzle_z,
+                       Tegra::Texture::SwizzleSource swizzle_w);
+
 private:
     void UploadGLMipmapTexture(u32 mip_map, GLuint read_fb_handle, GLuint draw_fb_handle);
 
@@ -394,6 +399,7 @@ private:
     GLenum gl_target{};
     GLenum gl_internal_format{};
     std::size_t cached_size_in_bytes{};
+    std::array<GLenum, 4> swizzle{GL_RED, GL_GREEN, GL_BLUE, GL_ALPHA};
 };
 
 class RasterizerCacheOpenGL final : public RasterizerCache<Surface> {
diff --git a/src/video_core/renderer_opengl/gl_state.cpp b/src/video_core/renderer_opengl/gl_state.cpp
index 997325efc..81af803bc 100644
--- a/src/video_core/renderer_opengl/gl_state.cpp
+++ b/src/video_core/renderer_opengl/gl_state.cpp
@@ -463,7 +463,8 @@ void OpenGLState::ApplyPolygonOffset() const {
 
 void OpenGLState::ApplyTextures() const {
     bool has_delta{};
-    std::size_t first{}, last{};
+    std::size_t first{};
+    std::size_t last{};
     std::array<GLuint, Tegra::Engines::Maxwell3D::Regs::NumTextureSamplers> textures;
 
     for (std::size_t i = 0; i < std::size(texture_units); ++i) {
@@ -478,16 +479,6 @@ void OpenGLState::ApplyTextures() const {
             }
             last = i;
         }
-
-        // Update the texture swizzle
-        if (textures[i] != 0 && (texture_unit.swizzle.r != cur_state_texture_unit.swizzle.r ||
-                                 texture_unit.swizzle.g != cur_state_texture_unit.swizzle.g ||
-                                 texture_unit.swizzle.b != cur_state_texture_unit.swizzle.b ||
-                                 texture_unit.swizzle.a != cur_state_texture_unit.swizzle.a)) {
-            std::array<GLint, 4> mask = {texture_unit.swizzle.r, texture_unit.swizzle.g,
-                                         texture_unit.swizzle.b, texture_unit.swizzle.a};
-            glTextureParameteriv(texture_unit.texture, GL_TEXTURE_SWIZZLE_RGBA, mask.data());
-        }
     }
 
     if (has_delta) {
@@ -498,7 +489,8 @@ void OpenGLState::ApplyTextures() const {
 
 void OpenGLState::ApplySamplers() const {
     bool has_delta{};
-    std::size_t first{}, last{};
+    std::size_t first{};
+    std::size_t last{};
     std::array<GLuint, Tegra::Engines::Maxwell3D::Regs::NumTextureSamplers> samplers;
     for (std::size_t i = 0; i < std::size(samplers); ++i) {
         samplers[i] = texture_units[i].sampler;
diff --git a/src/video_core/renderer_opengl/gl_state.h b/src/video_core/renderer_opengl/gl_state.h
index a5a7c0920..ced602bf6 100644
--- a/src/video_core/renderer_opengl/gl_state.h
+++ b/src/video_core/renderer_opengl/gl_state.h
@@ -127,19 +127,9 @@ public:
         GLuint texture; // GL_TEXTURE_BINDING_2D
         GLuint sampler; // GL_SAMPLER_BINDING
         GLenum target;
-        struct {
-            GLint r; // GL_TEXTURE_SWIZZLE_R
-            GLint g; // GL_TEXTURE_SWIZZLE_G
-            GLint b; // GL_TEXTURE_SWIZZLE_B
-            GLint a; // GL_TEXTURE_SWIZZLE_A
-        } swizzle;
 
         void Unbind() {
             texture = 0;
-            swizzle.r = GL_RED;
-            swizzle.g = GL_GREEN;
-            swizzle.b = GL_BLUE;
-            swizzle.a = GL_ALPHA;
         }
 
         void Reset() {
diff --git a/src/video_core/renderer_opengl/renderer_opengl.cpp b/src/video_core/renderer_opengl/renderer_opengl.cpp
index 761dd6be3..5b09c38ea 100644
--- a/src/video_core/renderer_opengl/renderer_opengl.cpp
+++ b/src/video_core/renderer_opengl/renderer_opengl.cpp
@@ -320,7 +320,6 @@ void RendererOpenGL::DrawScreenTriangles(const ScreenInfo& screen_info, float x,
     }};
 
     state.texture_units[0].texture = screen_info.display_texture;
-    state.texture_units[0].swizzle = {GL_RED, GL_GREEN, GL_BLUE, GL_ALPHA};
     // Workaround brigthness problems in SMO by enabling sRGB in the final output
     // if it has been used in the frame. Needed because of this bug in QT: QTBUG-50987
     state.framebuffer_srgb.enabled = OpenGLState::GetsRGBUsed();