在 iOS 上渲染到 CVPixelBuffer

Question

我有一个颤振插件，我需要在 iOS 上进行一些基本的 3D 渲染。 我决定使用 Metal API，因为 OpenGL ES 在平台上已被弃用。

在实现插件之前，我在 iOS 应用程序中实现了渲染。那里的渲染工作没有问题。

在渲染纹理时，整个区域都被黑色填充。

//preparation
Vertices = [Vertex(x:  1, y: -1,   tx: 1, ty: 1),
            Vertex(x: 1, y:  1,   tx: 1, ty: 0),
            Vertex(x: -1, y:  1,  tx: 0, ty: 0),
            Vertex(x:  -1, y:  -1,  tx: 0, ty: 1),]
Indices = [0, 1, 2, 2, 3, 0]

let d = [
    kCVPixelBufferOpenGLCompatibilityKey : true,
    kCVPixelBufferMetalCompatibilityKey : true
]
var cvret = CVPixelBufferCreate(kCFAllocatorDefault, width, height, kCVPixelFormatType_32BGRA, d as CFDictionary, &pixelBuffer); //FIXME jaki format
if(cvret != kCVReturnSuccess) {
    print("faield to create pixel buffer")
}

metalDevice = MTLCreateSystemDefaultDevice()! 

let desc = MTLTextureDescriptor.texture2DDescriptor(pixelFormat: MTLPixelFormat.rgba8Unorm, width: width, height: height, mipmapped: false)
desc.usage = MTLTextureUsage.renderTarget.union( MTLTextureUsage.shaderRead )
targetTexture = metalDevice.makeTexture(descriptor: desc)
metalCommandQueue = metalDevice.makeCommandQueue()!  
ciCtx = CIContext.init(mtlDevice: metalDevice)

let vertexBufferSize = Vertices.size()
vertexBuffer = metalDevice.makeBuffer(bytes: &Vertices, length: vertexBufferSize, options: .storageModeShared)

let indicesBufferSize = Indices.size()
indicesBuffer = metalDevice.makeBuffer(bytes: &Indices, length: indicesBufferSize, options: .storageModeShared)

let defaultLibrary = metalDevice.makeDefaultLibrary()!
let txProgram = defaultLibrary.makeFunction(name: "basic_fragment")
let vertexProgram = defaultLibrary.makeFunction(name: "basic_vertex") 

let pipelineStateDescriptor = MTLRenderPipelineDescriptor()
pipelineStateDescriptor.sampleCount = 1
pipelineStateDescriptor.vertexFunction = vertexProgram
pipelineStateDescriptor.fragmentFunction = txProgram
pipelineStateDescriptor.colorAttachments[0].pixelFormat = .rgba8Unorm

pipelineState = try! metalDevice.makeRenderPipelineState(descriptor: pipelineStateDescriptor)

//drawing
let renderPassDescriptor = MTLRenderPassDescriptor() 
renderPassDescriptor.colorAttachments[0].texture = targetTexture 
renderPassDescriptor.colorAttachments[0].loadAction = .clear 
renderPassDescriptor.colorAttachments[0].clearColor = MTLClearColor(red: 0.85, green: 0.85, blue: 0.85, alpha: 0.5) 
renderPassDescriptor.colorAttachments[0].storeAction = MTLStoreAction.store
renderPassDescriptor.renderTargetWidth = width
renderPassDescriptor.renderTargetHeight = height

guard let commandBuffer = metalCommandQueue.makeCommandBuffer() else { return }

guard let renderEncoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { return }
renderEncoder.label = "Offscreen render pass"
renderEncoder.setVertexBuffer(vertexBuffer, offset: 0, index: 0) 

renderEncoder.setRenderPipelineState(pipelineState)
renderEncoder.drawIndexedPrimitives(type: .triangle, indexCount: Indices.count, indexType: .uint32, indexBuffer: indicesBuffer, indexBufferOffset: 0) // 2

renderEncoder.endEncoding() 
commandBuffer.commit() 

//copy to pixel buffer
guard let img = CIImage(mtlTexture: targetTexture) else { return }
ciCtx.render(img, to: pixelBuffer!)

Answer 1

我很确定创建一个单独的 MTLTexture 然后将其转换为 CVPixelBuffer 不是一种可行的方法。您基本上是将其写入MTLTexture，然后仅使用该结果将其写入CIImage。

相反，您可以通过创建CVPixelBuffer 和CVPixelBufferCreateWithIOSurface 以及对应的MTLTexture 和makeTexture(descriptor:iosurface:plane:) ，让他们在下面共享IOSurface。

或者您可以创建一个与CVPixelBuffer 相同的内存别名的MTLBuffer，然后从该MTLBuffer 创建一个MTLTexture。如果您打算使用这种方法，我建议您也使用MTLBlitCommandEncoders 方法optimizeContentsForCPUAccess 和optimizeContentsForGPUAccess。您首先optimizeContentsForGPUAccess，然后使用GPU 上的纹理，然后使用optimizeContentsForCPUAccess 将像素旋转回CPU 可读格式。这样，您在渲染到纹理时不会损失性能。