我正在使用Core Image框架来检测VIPER架构中的微笑。

ViewController

extension ViewController: AVCaptureDataOutputSynchronizerDelegate {
    func dataOutputSynchronizer(_ synchronizer: AVCaptureDataOutputSynchronizer, didOutput synchronizedDataCollection: AVCaptureSynchronizedDataCollection) {
        guard let syncedDepthData: AVCaptureSynchronizedDepthData =
                synchronizedDataCollection.synchronizedData(for: multifaceCamera.depthDataOutput) as? AVCaptureSynchronizedDepthData,
              let videoDataOutput = self.videoDataOutput,
              let syncedVideoData: AVCaptureSynchronizedSampleBufferData =
                synchronizedDataCollection.synchronizedData(for: videoDataOutput) as? AVCaptureSynchronizedSampleBufferData else {
            return
        }
        
        if syncedDepthData.depthDataWasDropped || syncedVideoData.sampleBufferWasDropped {
            return
        }
        
        let depthData = syncedDepthData.depthData
        let _depthData = depthData.converting(toDepthDataType: kCVPixelFormatType_DepthFloat16)
        let depthPixelBuffer = _depthData.depthDataMap
        let sampleBuffer = syncedVideoData.sampleBuffer
        guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
            return
        }

        Task {
            await presenter.sendOutput(output: imageBuffer, depthPixelBuffer: depthPixelBuffer)
        }
    }
}

视图控制器捕获样本缓冲区和深度数据，并将它们发送到呈现器。深度数据来自TrueDepth相机，但由于不相关，在此示例代码中未示出其使用。

主讲人

func sendOutput(output: CVPixelBuffer, depthPixelBuffer: CVPixelBuffer?) async {
    await interactor.sendOutput(output: output, depthPixelBuffer: depthPixelBuffer)
}

presenter类中的sendOutput方法将数据转发给interactor。

互动者

func sendOutput(output: CVPixelBuffer, depthPixelBuffer: CVPixelBuffer?) async {
    let image = CIImage(cvPixelBuffer: output)
    let result = await isSmiling(image: image)
    /// Do something the result
}

func isSmiling(image: CIImage) async -> Bool {
    let detector = CIDetector(ofType: CIDetectorTypeFace, context: nil, options: [CIDetectorAccuracy: CIDetectorAccuracyHigh, CIDetectorSmile: true])
    let faces = detector?.features(in: image, options: [CIDetectorSmile : true as AnyObject]) as? [CIFaceFeature]
    if let facesFound = faces {
        for face in facesFound {
            return face.hasSmile
        }
    } else {
        return false
    }
    
    return false
}

interactor类中的CIDetector使用演示者发送的图像缓冲区来检测人脸中的微笑状态。我遇到的问题是detector?.features(in: image, options: [CIDetectorSmile : true as AnyObject]) as? [CIFaceFeature]不产生任何结果。我不是说它产生零。该方法被调用几次，然后它简单地停止，好像意识到特征方法没有产生任何结果或任务正在完成。不会崩溃，不会优雅地产生nils或错误，图像缓冲区只是停止从视图控制器产生。日志记录显示已正确创建检测器对象let detector = CIDetector(ofType: CIDetectorTypeFace, context: nil, options: [CIDetectorAccuracy: CIDetectorAccuracyHigh, CIDetectorSmile: true])。
为了测试图像缓冲区是否正确创建，我尝试使用Vision的VNDetectFaceLandmarksRequest来检测人脸，结果证明效果非常好。
令人困惑的是，如果我在视图控制器中使用相同的isSmiling方法，它可以正常工作，如下面的示例所示，这让我怀疑这是否与Task和async-await有关（尽管Vision的面部检测在交互器中工作正常）。

ViewController

extension ViewController: AVCaptureDataOutputSynchronizerDelegate {
    func dataOutputSynchronizer(_ synchronizer: AVCaptureDataOutputSynchronizer, didOutput synchronizedDataCollection: AVCaptureSynchronizedDataCollection) {
        guard let syncedDepthData: AVCaptureSynchronizedDepthData =
                synchronizedDataCollection.synchronizedData(for: multifaceCamera.depthDataOutput) as? AVCaptureSynchronizedDepthData,
              let videoDataOutput = self.videoDataOutput,
              let syncedVideoData: AVCaptureSynchronizedSampleBufferData =
                synchronizedDataCollection.synchronizedData(for: videoDataOutput) as? AVCaptureSynchronizedSampleBufferData else {
            return
        }
        
        if syncedDepthData.depthDataWasDropped || syncedVideoData.sampleBufferWasDropped {
            return
        }
        
        let depthData = syncedDepthData.depthData
        let _depthData = depthData.converting(toDepthDataType: kCVPixelFormatType_DepthFloat16)
        let depthPixelBuffer = _depthData.depthDataMap
        let sampleBuffer = syncedVideoData.sampleBuffer
        guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
            return
        }

        Task {
           let image = CIImage(cvPixelBuffer: imageBuffer)
           let result = await isSmiling(image: image)
           /// Do something the result
        }
    }
}

我在Xcode控制台得到的唯一错误消息是：
输入Tensor宽度必须与64字节对齐。
我甚至不确定它是否与CIDetector有关。
我尝试设置AVCaptureVideoDataOutput如下

videoDataOutput.videoSettings = [kCVPixelBufferPixelFormatTypeKey as String: Int(kCVPixelFormatType_64RGBALE)]
/// or kCVPixelFormatType_64RGBALE

或者甚至选择其宽度可被16或64整除的可用格式，并将其分配给设备的活动格式，考虑到给定每个像素是4字节，以像素为单位的图像宽度需要是16的倍数，以字节为单位的总宽度是64的倍数：

for format in availableFormats {
    let dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription)
    if dimensions.width % 16 == 0 {
        byteAlignedFormats.append(format)
    }
}

/// assign one of the byteAlignedFormats to be the device's active format

let deviceInput = try AVCaptureDeviceInput(device: device)
try device.lockForConfiguration()
device.activeFormat = byteAlignedFormat
device.unlockForConfiguration()

不幸的是，上述尝试都没有奏效。