[draft] Fix obscure render corruption

minecraft/src/main/kotlin/org/polyfrost/oneconfig/internal/ui/compose/SkiaCtx.kt

@@ -70,6 +70,12 @@ object SkiaCtx {
     private var composeSurface: Surface? = null
     private var composeBrt: BackendRenderTarget? = null
 
+    private var hudNeedsSamplingTransition = false
+    private var composeNeedsSamplingTransition = false
+
+    private var hudRealIsGeneral = false
+    private var composeRealIsGeneral = false
+
     @Volatile
     private var composeActive = false
     @Volatile
@@ -230,7 +236,10 @@ object SkiaCtx {
         val draws = queuedHudDraws.toList()
         queuedHudDraws.clear()
         if (draws.isEmpty()) return
-        flushToTarget(draws, resolveHudSurface() ?: return)
+        val surface = resolveHudSurface() ?: return
+        if (hudRealIsGeneral) hudTarget?.let { vulkanService?.transitionOffscreenForRendering(it) }
+        flushToTarget(draws, surface)
+        hudNeedsSamplingTransition = true
     }
 
     @Volatile
@@ -263,7 +272,10 @@ object SkiaCtx {
         val queued = queuedDraws.toList()
         queuedDraws.clear()
         val draws = queued + { block.run() }
-        flushToTarget(draws, resolveComposeSurface() ?: return)
+        val surface = resolveComposeSurface() ?: return
+        if (composeRealIsGeneral) composeTarget?.let { vulkanService?.transitionOffscreenForRendering(it) }
+        flushToTarget(draws, surface)
+        composeNeedsSamplingTransition = true
         blitCompose(ctx)
     }
 
@@ -313,7 +325,7 @@ object SkiaCtx {
 
     fun blitHud(guiGraphics: GuiGraphicsExtractor) {
         val rt = hudTarget ?: return
-        val w = rt.width;
+        val w = rt.width
         val h = rt.height
         val guiScale = client.window.guiScale.toFloat()
 
@@ -328,6 +340,11 @@ object SkiaCtx {
         wrapper.setGpuTexture(colorTex)
         //? >= 1.21.8 {
         wrapper.setGpuTextureView(rt.getColorTextureView())
+        if (hudNeedsSamplingTransition) {
+            vulkanService?.transitionOffscreenForSampling(rt)
+            hudNeedsSamplingTransition = false
+            hudRealIsGeneral = true
+        }
         guiGraphics.pose().pushMatrix()
         guiGraphics.pose().scale(1f / guiScale, 1f / guiScale)
         guiGraphics.blit(net.minecraft.client.renderer.RenderPipelines.GUI_TEXTURED, HUD_TEXTURE_LOC, 0, 0, 0f, 0f, w, h, w, h)
@@ -377,6 +394,11 @@ object SkiaCtx {
         wrapper.setGpuTexture(colorTex)
         //? >= 1.21.8 {
         wrapper.setGpuTextureView(rt.getColorTextureView())
+        if (composeNeedsSamplingTransition) {
+            vulkanService?.transitionOffscreenForSampling(rt)
+            composeNeedsSamplingTransition = false
+            composeRealIsGeneral = true
+        }
         guiGraphics.pose().pushMatrix()
         guiGraphics.pose().scale(1f / guiScale, 1f / guiScale)
         guiGraphics.blit(net.minecraft.client.renderer.RenderPipelines.GUI_TEXTURED, COMPOSE_TEXTURE_LOC, 0, 0, 0f, 0f, w, h, w, h)
@@ -471,6 +493,7 @@ object SkiaCtx {
 
             if (isVulkanMode) {
                 directContext.flushAndSubmit(mainSurface, false)
+                vulkanService?.restoreMainRTLayout()
             } else {
                 directContext.flush()
 //                if (profiling) {
@@ -546,7 +569,8 @@ object SkiaCtx {
         if (w <= 0 || h <= 0) return null
 
         var rt = hudTarget
-        if (rt == null || rt.width != w || rt.height != h) {
+        val needNewTarget = rt == null || rt.width != w || rt.height != h
+        if (needNewTarget) {
             destroyHudTarget()
             //? if >= 26.2 {
             /*rt = TextureTarget(null, w, h, true, com.mojang.blaze3d.GpuFormat.RGBA8_UNORM)
@@ -559,7 +583,6 @@ object SkiaCtx {
             *///? }
             hudTarget = rt
 
-            val svc = vulkanService ?: return null
             //? >= 1.21.5 {
             if (!isVulkanMode) {
                 val fboId = org.polyfrost.oneconfig.internal.ui.RenderTargetFbo.getFboId(rt)
@@ -571,6 +594,13 @@ object SkiaCtx {
                 }
             }
             //? }
+        }
+
+        if (needNewTarget || hudSurface == null) {
+            hudSurface?.close(); hudSurface = null
+            hudBrt?.close(); hudBrt = null
+            hudRealIsGeneral = false
+            val svc = vulkanService ?: return null
             val (brt, colorFmt) = svc.makeOffscreenBRT(rt, w, h)
             hudBrt = brt
             hudSurface = Surface.makeFromBackendRenderTarget(
@@ -601,7 +631,8 @@ object SkiaCtx {
         if (w <= 0 || h <= 0) return null
 
         var rt = composeTarget
-        if (rt == null || rt.width != w || rt.height != h) {
+        val needNewTarget = rt == null || rt.width != w || rt.height != h
+        if (needNewTarget) {
             destroyComposeTarget()
             //? if >= 26.2 {
             /*rt = TextureTarget(null, w, h, true, com.mojang.blaze3d.GpuFormat.RGBA8_UNORM)
@@ -614,7 +645,6 @@ object SkiaCtx {
             *///? }
             composeTarget = rt
 
-            val svc = vulkanService ?: return null
             //? >= 1.21.5 {
             if (!isVulkanMode) {
                 val fboId = org.polyfrost.oneconfig.internal.ui.RenderTargetFbo.getFboId(rt)
@@ -626,7 +656,14 @@ object SkiaCtx {
                 }
             }
             //? }
-            val (brt, colorFmt) = svc.makeOffscreenBRT(rt, w, h)
+        }
+
+        if (needNewTarget || composeSurface == null) {
+            composeSurface?.close(); composeSurface = null
+            composeBrt?.close(); composeBrt = null
+            composeRealIsGeneral = false
+            val svc = vulkanService ?: return null
+            val (brt, colorFmt) = svc.makeOffscreenBRT(rt!!, w, h)
             composeBrt = brt
             val composeOrigin = if (isVulkanMode) SurfaceOrigin.TOP_LEFT else SurfaceOrigin.BOTTOM_LEFT
             composeSurface = Surface.makeFromBackendRenderTarget(
@@ -701,7 +738,7 @@ object SkiaCtx {
         val h = client.window.height
         if (w <= 0 || h <= 0) return null
 
-        if (w != vkSurfaceWidth || h != vkSurfaceHeight) {
+        if (w != vkSurfaceWidth || h != vkSurfaceHeight || svc.offscreenNeedsPerFrameRewrap) {
             invalidateVkSurfaces()
             vkSurfaceWidth = w
             vkSurfaceHeight = h

minecraft/src/main/kotlin/org/polyfrost/oneconfig/internal/ui/services/NativeVulkanService.kt

@@ -10,8 +10,10 @@ import net.minecraft.client.Minecraft
 import org.jetbrains.skia.BackendRenderTarget
 import org.jetbrains.skia.DirectContext
 import org.jetbrains.skia.SurfaceColorFormat
+import org.lwjgl.system.MemoryStack
 import org.lwjgl.vulkan.VK
 import org.lwjgl.vulkan.VK12.*
+import org.lwjgl.vulkan.VkImageMemoryBarrier
 import org.polyfrost.oneconfig.internal.mixin.blaze3d.GpuDeviceAccessor
 import org.slf4j.LoggerFactory
 
@@ -31,6 +33,8 @@ class NativeVulkanService private constructor(
 
     override val isVulkan = true
 
+    override val offscreenNeedsPerFrameRewrap = true
+
     override fun makeDirectContext(): DirectContext {
         val provider = VK.getFunctionProvider()
         val instanceProcAddr = provider.getFunctionAddress("vkGetInstanceProcAddr")
@@ -51,7 +55,7 @@ class NativeVulkanService private constructor(
         width, height,
         vkImageHandle,
         VK_IMAGE_TILING_OPTIMAL,
-        VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
+        VK_IMAGE_LAYOUT_GENERAL,
         vkFormat,
         VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT or VK_IMAGE_USAGE_TRANSFER_DST_BIT
                 or VK_IMAGE_USAGE_SAMPLED_BIT or VK_IMAGE_USAGE_TRANSFER_SRC_BIT,
@@ -86,11 +90,86 @@ class NativeVulkanService private constructor(
         Triple(0L, 0, 0)
     }
 
+    override fun restoreMainRTLayout() {
+        transitionImage(
+            Minecraft.getInstance().gameRenderer.mainRenderTarget().colorTexture as? VulkanGpuTexture,
+            oldLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
+            newLayout = VK_IMAGE_LAYOUT_GENERAL,
+            srcStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
+            dstStageMask = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
+            srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT,
+            dstAccessMask = VK_ACCESS_MEMORY_READ_BIT or VK_ACCESS_MEMORY_WRITE_BIT,
+        )
+    }
+
+    override fun transitionOffscreenForSampling(target: RenderTarget) {
+        transitionImage(
+            target.colorTexture as? VulkanGpuTexture,
+            oldLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
+            newLayout = VK_IMAGE_LAYOUT_GENERAL,
+            srcStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
+            dstStageMask = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
+            srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT,
+            dstAccessMask = VK_ACCESS_SHADER_READ_BIT,
+        )
+    }
+
+    override fun transitionOffscreenForRendering(target: RenderTarget) {
+        transitionImage(
+            target.colorTexture as? VulkanGpuTexture,
+            oldLayout = VK_IMAGE_LAYOUT_GENERAL,
+            newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
+            srcStageMask = VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
+            dstStageMask = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
+            srcAccessMask = VK_ACCESS_SHADER_READ_BIT,
+            dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT or VK_ACCESS_COLOR_ATTACHMENT_READ_BIT,
+        )
+    }
+
     override fun midFrameFlush() {
         // Submit MC's pending command buffer so Skia sees a consistent image state.
         RenderSystem.getDevice().createCommandEncoder().submit()
     }
 
+    private fun transitionImage(
+        tex: VulkanGpuTexture?,
+        oldLayout: Int, newLayout: Int,
+        srcStageMask: Int, dstStageMask: Int,
+        srcAccessMask: Int, dstAccessMask: Int,
+    ) {
+        if (tex == null) return
+        try {
+            val device = (RenderSystem.getDevice() as? GpuDeviceAccessor)?.`oneconfig$getBackend`() as? VulkanDevice ?: return
+            val encoder = device.createCommandEncoder()
+            val cmd = encoder.allocateAndBeginTransientCommandBuffer()
+            try {
+                MemoryStack.stackPush().use { stack ->
+                    val barrier = VkImageMemoryBarrier.calloc(1, stack)
+                        .sType(VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER)
+                        .oldLayout(oldLayout)
+                        .newLayout(newLayout)
+                        .srcAccessMask(srcAccessMask)
+                        .dstAccessMask(dstAccessMask)
+                        .srcQueueFamilyIndex(VK_QUEUE_FAMILY_IGNORED)
+                        .dstQueueFamilyIndex(VK_QUEUE_FAMILY_IGNORED)
+                        .image(tex.vkImage())
+                    barrier.subresourceRange()
+                        .aspectMask(VK_IMAGE_ASPECT_COLOR_BIT)
+                        .baseMipLevel(0).levelCount(1).baseArrayLayer(0).layerCount(1)
+                    vkCmdPipelineBarrier(
+                        cmd, srcStageMask, dstStageMask, 0,
+                        null, null, barrier,
+                    )
+                }
+            } finally {
+                vkEndCommandBuffer(cmd)
+                encoder.execute(cmd)
+            }
+        } catch (e: Exception) {
+            LOG.warn("transitionImage failed", e)
+        }
+    }
+
     companion object {
         private val LOG = LoggerFactory.getLogger(NativeVulkanService::class.java)
 

minecraft/src/main/kotlin/org/polyfrost/oneconfig/internal/ui/services/VulkanService.kt

@@ -18,6 +18,14 @@ interface VulkanService {
      */
     val usesDeferredCompose: Boolean get() = false
 
+    val offscreenNeedsPerFrameRewrap: Boolean get() = false
+
+    fun transitionOffscreenForSampling(target: RenderTarget) {}
+
+    fun transitionOffscreenForRendering(target: RenderTarget) {}
+
+    fun restoreMainRTLayout() {}
+
     fun makeDirectContext(): DirectContext
 
     /**