AVCapturePhoto SemanticSegmentationMatte nil 没有音频输入？答案

【问题标题】：AVCapturePhoto SemanticSegmentationMatte nil without audio input?AVCapturePhoto SemanticSegmentationMatte nil 没有音频输入？
【发布时间】：2020-11-10 05:38:51
【问题描述】：

当我将音频输入添加到捕获会话时，photoOutput(_ output: AVCapturePhotoOutput, didFinishProcessingPhoto photo: AVCapturePhoto, error: Error?) 回调会正确返回语义分割遮罩。没有音频输入，返回的遮罩为零。是否可以避免添加音频输入并要求用户授予麦克风权限以获得遮罩？

    // MARK: - Session

private func setupSession() {
    captureSession = AVCaptureSession()
    captureSession?.sessionPreset = .photo
    setupInputOutput()
    setupPreviewLayer(view)
    captureSession?.startRunning()
}
// MARK: - Settings

private func setupCamera() {
    
    settings = AVCapturePhotoSettings()
    
    let supportsHEVC = AVAssetExportSession.allExportPresets().contains(AVAssetExportPresetHEVCHighestQuality)

    settings = supportsHEVC ? AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCodecType.hevc]) : AVCapturePhotoSettings(format: [AVVideoCodecKey: AVVideoCodecType.jpeg])
    
    settings!.flashMode = .auto
    settings!.isHighResolutionPhotoEnabled = true
    settings!.previewPhotoFormat = [kCVPixelBufferPixelFormatTypeKey as String: settings!.__availablePreviewPhotoPixelFormatTypes.first ?? NSNumber()]
    settings!.isDepthDataDeliveryEnabled = true
    settings!.isPortraitEffectsMatteDeliveryEnabled = true
    if self.photoOutput?.enabledSemanticSegmentationMatteTypes.isEmpty == false {
        settings!.enabledSemanticSegmentationMatteTypes = self.photoOutput?.enabledSemanticSegmentationMatteTypes ?? [AVSemanticSegmentationMatte.MatteType]()
    }

    settings!.photoQualityPrioritization = self.photoQualityPrioritizationMode
}

private func setupInputOutput() {
    photoOutput = AVCapturePhotoOutput()
    
    guard let captureSession = captureSession  else { return }
    guard let photoOutput = photoOutput else { return }
    
    do {
        captureSession.beginConfiguration()
        captureSession.sessionPreset = .photo
        let devices = self.videoDeviceDiscoverySession.devices
        currentDevice = devices.first(where: { $0.position == .front && $0.deviceType == .builtInTrueDepthCamera })

        guard let videoDevice = currentDevice else {
            captureSession.commitConfiguration()
            return
        }
        
        videoDeviceInput = try AVCaptureDeviceInput(device: videoDevice)

        if captureSession.canAddInput(videoDeviceInput) {
            captureSession.addInput(videoDeviceInput)
        } else {
            captureSession.commitConfiguration()
            return
        }
        
        currentDevice = AVCaptureDevice.default(for: .audio)
        captureDeviceInput = try AVCaptureDeviceInput(device: currentDevice!)

        if captureSession.canAddInput(captureDeviceInput) {
            captureSession.addInput(captureDeviceInput)
        } else {
            captureSession.commitConfiguration()
            return
        }
    } catch {
        errorMessage = error.localizedDescription
        print(error.localizedDescription)
        captureSession.commitConfiguration()
        return
    }

    if captureSession.canAddOutput(photoOutput) {
        captureSession.addOutput(photoOutput)

        photoOutput.isHighResolutionCaptureEnabled = true
        photoOutput.isLivePhotoCaptureEnabled = photoOutput.isLivePhotoCaptureSupported
        photoOutput.isDepthDataDeliveryEnabled = photoOutput.isDepthDataDeliverySupported
        photoOutput.isPortraitEffectsMatteDeliveryEnabled = photoOutput.isPortraitEffectsMatteDeliverySupported
        photoOutput.enabledSemanticSegmentationMatteTypes = photoOutput.availableSemanticSegmentationMatteTypes
      
        photoOutput.maxPhotoQualityPrioritization = .balanced
    }
    captureSession.commitConfiguration()
}

private func setupPreviewLayer(_ view: UIView) {
    self.cameraPreviewLayer = AVCaptureVideoPreviewLayer(session: captureSession ?? AVCaptureSession())
    self.cameraPreviewLayer?.videoGravity = AVLayerVideoGravity.resizeAspectFill
    self.cameraPreviewLayer?.connection?.videoOrientation = AVCaptureVideoOrientation.portrait
    self.cameraPreviewLayer?.frame = view.frame
    view.layer.insertSublayer(self.cameraPreviewLayer ?? AVCaptureVideoPreviewLayer(), at: 0)
}

【问题讨论】：

是的，绝对应该。我只能想象，当您在要录制视频的情况下添加音频输出时，会话会更改内部设置。你能分享你的会话设置代码吗？
谢谢回复。我添加了设置代码。此代码有效，但是当我注释掉为音频添加 AVCaptureDeviceInput 的行时，回调为分段遮罩提供 nil。请注意，我首先调用 setupSession()，然后调用 setupCamera()。
我的猜测是添加音频输入会将sessionPreset 更改为某种视频格式，这将更改遮罩的传递设置（因为视频不支持分段遮罩）。我认为您需要针对不同的用例（人像照片与视频录制）进行两种不同的配置。

标签： ios avfoundation avcapturesession semantic-segmentation

【解决方案1】：

无论是否设置音频输入，我都无法返回语义分割遮罩 (SSM)。我目前正在 iPhone X 上进行开发。在挣扎了一段时间后，我在 WWDC2021 期间的 1-1 实验室会议上向 Apple 提出了这个问题。有人告诉我，该 API 只会使我的设备可以看到人像效果遮罩。 iPhone 11 及更高版本将能够获得皮肤、牙齿和头发。他们最近在没有宣布的情况下潜入的新眼镜 ssm 需要 iPhone 12。

【讨论】：