ARKit相机框架中的纹理ARMeshGeometry？

Question

这个问题在某种程度上建立在this 帖子上，其中的想法是从带有激光雷达扫描仪的 iOS 设备中获取ARMeshGeometry，计算纹理坐标，并将采样的相机帧应用为给定网格的纹理，从而允许用户创建其环境的“逼真的”3D 表示。

根据那篇帖子，我已经调整了其中一个响应来计算纹理坐标，就像这样；

func buildGeometry(meshAnchor: ARMeshAnchor, arFrame: ARFrame) -> SCNGeometry {
    let vertices = meshAnchor.geometry.vertices

    let faces = meshAnchor.geometry.faces
    let camera = arFrame.camera
    let size = arFrame.camera.imageResolution
    
    // use the MTL buffer that ARKit gives us
    let vertexSource = SCNGeometrySource(buffer: vertices.buffer, vertexFormat: vertices.format, semantic: .vertex, vertexCount: vertices.count, dataOffset: vertices.offset, dataStride: vertices.stride)
    
    // set the camera matrix
    let modelMatrix = meshAnchor.transform
    
    var textCords = [CGPoint]()
    for index in 0..<vertices.count {
        let vertexPointer = vertices.buffer.contents().advanced(by: vertices.offset + vertices.stride * index)
        let vertex = vertexPointer.assumingMemoryBound(to: (Float, Float, Float).self).pointee
        let vertex4 = SIMD4<Float>(vertex.0, vertex.1, vertex.2, 1)
        let world_vertex4 = simd_mul(modelMatrix, vertex4)
        let world_vector3 = simd_float3(x: world_vertex4.x, y: world_vertex4.y, z: world_vertex4.z)
        let pt = camera.projectPoint(world_vector3, orientation: .portrait, viewportSize: CGSize(width: CGFloat(size.height), height: CGFloat(size.width)))
        let v = 1.0 - Float(pt.x) / Float(size.height)
        let u = Float(pt.y) / Float(size.width)
        
        //let z = vector_float2(u, v)
        let c = CGPoint(x: v, y: u)
        textCords.append(c)
    }
    
    // Setup the texture coordinates
    let textureSource = SCNGeometrySource(textureCoordinates: textCords)
    
    // Setup the normals
    let normalsSource = SCNGeometrySource(meshAnchor.geometry.normals, semantic: .normal)
    
    // Setup the geometry
    let faceData = Data(bytesNoCopy: faces.buffer.contents(), count: faces.buffer.length, deallocator: .none)
    let geometryElement = SCNGeometryElement(data: faceData, primitiveType: .triangles, primitiveCount: faces.count, bytesPerIndex: faces.bytesPerIndex)
    let nodeGeometry = SCNGeometry(sources: [vertexSource, textureSource, normalsSource], elements: [geometryElement])
    
    /* Setup texture - THIS IS WHERE I AM STUCK
    let texture = textureConverter.makeTextureForMeshModel(frame: arFrame)
    */
    
    let imageMaterial = SCNMaterial()
    imageMaterial.isDoubleSided = false
    imageMaterial.diffuse.contents = texture!
    nodeGeometry.materials = [imageMaterial]
    
    return nodeGeometry
}

我正在努力确定这些纹理坐标是否实际计算正确，以及随后如何对相机帧进行采样以将相关帧图像应用为该网格的纹理。

链接的问题表明将ARFrame 的capturedImage（这是CVPixelBuffer）属性转换为MTLTexture 将是实时性能的理想选择，但对我来说很明显CVPixelBuffer 是YCbCr 图像，而我相信我需要RGB 图像。

在我的textureConverter 类中，我试图将CVPixelBuffer 转换为MTLTexture，但我不确定如何返回RGB MTLTexture；

func makeTextureForMeshModel(frame: ARFrame) -> MTLTexture? {
    if CVPixelBufferGetPlaneCount(frame.capturedImage) < 2 {
        return nil
    }
    let cameraImageTextureY = createTexture(fromPixelBuffer: frame.capturedImage, pixelFormat: .r8Unorm, planeIndex: 0)
    let cameraImageTextureCbCr = createTexture(fromPixelBuffer: frame.capturedImage, pixelFormat: .rg8Unorm, planeIndex: 1)
    
    /* How do I blend the Y and CbCr textures, or return a RGB texture, to return a single MTLTexture?
    return ...
}

func createTexture(fromPixelBuffer pixelBuffer: CVPixelBuffer, pixelFormat: MTLPixelFormat, planeIndex: Int) -> CVMetalTexture? {
    let width = CVPixelBufferGetWidthOfPlane(pixelBuffer, planeIndex)
    let height = CVPixelBufferGetHeightOfPlane(pixelBuffer, planeIndex)
    
    var texture: CVMetalTexture? = nil
    let status = CVMetalTextureCacheCreateTextureFromImage(nil, textureCache, pixelBuffer, nil, pixelFormat,
                                                           width, height, planeIndex, &texture)
    
    if status != kCVReturnSuccess {
        texture = nil
    }
    
    return texture
}

最后，我不完全确定我是否真的需要 RGB 纹理与 YCbCr 纹理，但我仍然不确定如何返回正确的图像纹理（我尝试只返回 CVPixelBuffer 而不必担心 YCbCr 色彩空间，通过手动设置纹理格式，结果是一个非常奇怪的图像）。

Answer 1

您可以在这里查看我的存储库：MetalWorldTextureScan

该项目演示了如何：

• 在扫描时使用 Metal 渲染网格
• 将扫描内容裁剪到边界框
• 保存相机帧以进行纹理处理
• 从您的扫描中创建和纹理化 SCNGeometry

计算纹理坐标：

帧：保存的帧用于纹理

vert：要投影到框架中的顶点

aTrans：顶点所在的网格“块”的变换

func getTextureCoord(frame: ARFrame, vert: SIMD3<Float>, aTrans: simd_float4x4) -> vector_float2 {
    
    // convert vertex to world coordinates
    let cam = frame.camera
    let size = cam.imageResolution
    let vertex4 = vector_float4(vert.x, vert.y, vert.z, 1)
    let world_vertex4 = simd_mul(aTrans, vertex4)
    let world_vector3 = simd_float3(x: world_vertex4.x, y: world_vertex4.y, z: world_vertex4.z)
    
    // project the point into the camera image to get u,v
    let pt = cam.projectPoint(world_vector3,
        orientation: .portrait,
        viewportSize: CGSize(
            width: CGFloat(size.height),
            height: CGFloat(size.width)))
    let v = 1.0 - Float(pt.x) / Float(size.height)
    let u = Float(pt.y) / Float(size.width)
    
    let tCoord = vector_float2(u, v)
    
    return tCoord
}

为纹理保存帧：

一个名为“TextureFrame”的结构用于保存位置、ARFrame 和其他可能有用的信息。

struct TextureFrame {
    var key: String       // date/time/anything
    var dist: CGFloat     // dist from bBox
    var frame: ARFrame    // saved frame
    var pos: SCNVector3   // location in reference to bBox
}

如何使用：

func saveTextureFrame() {
    guard let frame = session.currentFrame else {
        print("can't get current frame")
        return
    }
    
    let camTrans = frame.camera.transform
    let camPos = SCNVector3(camTrans.columns.3.x, camTrans.columns.3.y, camTrans.columns.3.z)
    let cam2BoxLocal = SCNVector3(camPos.x - bBoxOrigin.x, camPos.y - bBoxOrigin.y, camPos.z - bBoxOrigin.z)
    let dist = dist3D(a: camPos, b: bBoxOrigin)
    
    let dateFormatter = DateFormatter()
    dateFormatter.dateFormat = "yyyy:MM:dd:HH:mm:ss:SS"
    dateFormatter.timeZone = TimeZone(abbreviation: "CDT")
    let date = Date()
    let dString = dateFormatter.string(from: date)
    
    let textFrame = TextureFrame(key: dString, dist: dist, frame: frame, pos: cam2BoxLocal)
    textureCloud.append(textFrame)
    delegate.didSaveFrame(renderer: self)
}

制作纹理网格：

这发生在 ma​​keTexturedMesh() 函数中。我们遍历所有网格块，并遍历每个块的每个面（三角形），在此处计算纹理坐标，并为三角形创建单个 SCNGeometry 并添加到场景中。 recombineGeometries() 函数也被定义用于重新组合三角形。

func makeTexturedMesh() {
    
    let worldMeshes = renderer.worldMeshes
    let textureCloud = renderer.textureCloud
    
    print("texture images: \(textureImgs.count)")
    
    // each 'mesh' is a chunk of the whole scan
    for mesh in worldMeshes {
        
        let aTrans = SCNMatrix4(mesh.transform)
        
        let vertices: ARGeometrySource = mesh.vertices
        let normals: ARGeometrySource = mesh.normals
        let faces: ARGeometryElement = mesh.submesh
        
        var texture: UIImage!
        
        // a face is just a list of three indices, each representing a vertex
        for f in 0..<faces.count {
            
            // check to see if each vertex of the face is inside of our box
            var c = 0
            let face = face(at: f, faces: faces)
            for fv in face {
                // this is set by the renderer
                if mesh.inBox[fv] == 1 {
                    c += 1
                }
            }
            
            guard c == 3 else {continue}
            
            // all verts of the face are in the box, so the triangle is visible
            var fVerts: [SCNVector3] = []
            var fNorms: [SCNVector3] = []
            var tCoords: [vector_float2] = []
            
            // convert each vertex and normal to world coordinates
            // get the texture coordinates
            for fv in face {
                
                let vert = vertex(at: UInt32(fv), vertices: vertices)
                let vTrans = SCNMatrix4MakeTranslation(vert[0], vert[1], vert[2])
                let wTrans = SCNMatrix4Mult(vTrans, aTrans)
                let wPos = SCNVector3(wTrans.m41, wTrans.m42, wTrans.m43)
                fVerts.append(wPos)
                
                let norm = normal(at: UInt32(fv), normals: normals)
                let nTrans = SCNMatrix4MakeTranslation(norm[0], norm[1], norm[2])
                let wNTrans = SCNMatrix4Mult(nTrans, aTrans)
                let wNPos = SCNVector3(wNTrans.m41, wTrans.m42, wNTrans.m43)
                fNorms.append(wNPos)
                
                
                // here's where you would find the frame that best fits
                // for simplicity, just use the last frame here
                let tFrame = textureCloud.last!.frame
                let tCoord = getTextureCoord(frame: tFrame, vert: vert, aTrans: mesh.transform)
                tCoords.append(tCoord)
                texture = textureImgs[textureCloud.count - 1]
                
                // visualize the normals if you want
                if mesh.inBox[fv] == 1 {
                    //let normVis = lineBetweenNodes(positionA: wPos, positionB: wNPos, inScene: arView.scene)
                    //arView.scene.rootNode.addChildNode(normVis)
                }
            }
            allVerts.append(fVerts)
            allNorms.append(fNorms)
            allTCrds.append(tCoords)
            
            // make a single triangle mesh out each face
            let vertsSource = SCNGeometrySource(vertices: fVerts)
            let normsSource = SCNGeometrySource(normals: fNorms)
            let facesSource = SCNGeometryElement(indices: [UInt32(0), UInt32(1), UInt32(2)], primitiveType: .triangles)
            let textrSource = SCNGeometrySource(textureCoordinates: tCoords)
            let geom = SCNGeometry(sources: [vertsSource, normsSource, textrSource], elements: [facesSource])
            
            // texture it with a saved camera frame
            let mat = SCNMaterial()
            mat.diffuse.contents = texture
            mat.isDoubleSided = false
            geom.materials = [mat]
            let meshNode = SCNNode(geometry: geom)
            
            DispatchQueue.main.async {
                self.scanNode.addChildNode(meshNode)
            }
        }
    }
}

该项目还包括从文档目录保存和加载网格的方法。

这绝不是对 3D 扫描进行网格化和纹理化的最佳方式，但它很好地演示了如何开始使用内置 iOS 框架。