I update a version that runs without crash. But the prediction speed is almost the same as sync version API. The createFrameAsync is called from ScreenCaptureKit stream.
private func createFrameAsync(for sampleBuffer: CMSampleBuffer ) {
if let surface = getIOSurface(for: sampleBuffer) {
Task {
do {
try await runModelAsync(surface)
} catch {
os_log("error: \(error)")
}
}
}
}
func runModelAsync(_ surface: IOSurface) async throws {
try Task.checkCancellation()
guard let model = mlmodel else {return}
do {
// Resize input
var px: Unmanaged<CVPixelBuffer>?
let status = CVPixelBufferCreateWithIOSurface(kCFAllocatorDefault, surface, nil, &px)
guard status == kCVReturnSuccess, let px2 = px?.takeRetainedValue() else { return }
guard let data = resizeIOSurfaceIntoPixelBuffer(
of: px2,
from: CGRect(x: 0, y: 0, width: InputWidth, height: InputHeight)
) else { return }
// Model Prediction
var results: [Float] = []
let inferenceStartTime = Date()
let input = model_smallInput(input: data)
let prediction = try await model.model.prediction(from: input)
// Get result into format
if let output = prediction.featureValue(for: "output")?.multiArrayValue {
if let bufferPointer = try? UnsafeBufferPointer<Float>(output) {
results = Array(bufferPointer)
}
}
// Set Render Data for Metal Rendering
await ScreenRecorder.shared
.setRenderDataNormalized(surface: surface, depthData: results)
} catch {
print("Error performing inference: \(error)")
}
}
Since Async prediction API cannot speed up the prediction, is there anything else I can do? The prediction time is almost the same on macbook M2 Pro and macbook M1 Air!
