@Davidbaraff2 that's the code im currently using. I try to Capture faces with vision and the back camera in portrait Mode. To be able to render the bounding boxes on screen, I noticed using .leftMirrored as orientation for VNImageRequestHandler helps a lot, but I don't get DockKit to track the faces correctly.
What am I missing here?
class TestViewController: UIViewController {
private let captureSession = AVCaptureSession()
private let videoDataOutput = AVCaptureVideoDataOutput()
private lazy var previewLayer = AVCaptureVideoPreviewLayer(session: self.captureSession)
private var faceLayers: [CAShapeLayer] = []
private var dockAccessory: DockAccessory?
private var captureDevice: AVCaptureDevice?
override func viewDidLoad() {
super.viewDidLoad()
Task {
try await DockAccessoryManager.shared.setSystemTrackingEnabled(false)
for await accessory in try DockAccessoryManager.shared.accessoryStateChanges {
print(accessory.state)
DispatchQueue.main.async {
//self.connectedToDock = accessory.state == DockAccessory.State.docked
self.dockAccessory = accessory.accessory
}
}
}
setupCamera()
captureSession.startRunning()
func setupCamera() {
self.captureSession.sessionPreset = .hd1280x720
let deviceDiscoverySession = AVCaptureDevice.DiscoverySession(
deviceTypes: [
.builtInDualCamera,
.builtInTripleCamera
],
mediaType: .video,
position: .back
)
if let device = deviceDiscoverySession.devices.first {
if let deviceInput = try? AVCaptureDeviceInput(device: device) {
if captureSession.canAddInput(deviceInput) {
captureSession.addInput(deviceInput)
setupPreview()
}
}
self.captureDevice = device
}
func setupPreview() {
self.previewLayer.videoGravity = .resizeAspectFill
self.view.layer.addSublayer(self.previewLayer)
self.previewLayer.frame = self.view.frame
self.videoDataOutput.videoSettings = [
(kCVPixelBufferPixelFormatTypeKey as NSString) : NSNumber(value: kCVPixelFormatType_32BGRA)
] as [String : Any]
self.videoDataOutput.setSampleBufferDelegate(
self,
queue: DispatchQueue(label: "camera queue")
)
self.captureSession.addOutput(self.videoDataOutput)
let videoConnection = self.videoDataOutput.connection(with: .video)
videoConnection?.videoOrientation = .portrait
}
}
}
private var frameCounter = 0
private var lastTimestamp = Date()
}
extension TestViewController: AVCaptureVideoDataOutputSampleBufferDelegate {
func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from connection: AVCaptureConnection) {
guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
return
}
let faceDetectionRequest = VNDetectFaceLandmarksRequest(completionHandler: { (request: VNRequest, error: Error?) in
DispatchQueue.main.async {
self.faceLayers.forEach({ drawing in drawing.removeFromSuperlayer() })
if let observations = request.results as? [VNFaceObservation] {
self.handleFaceDetectionObservations(
observations: observations,
imageBuffer,
sampleBuffer
)
}
}
})
let imageRequestHandler = VNImageRequestHandler(cvPixelBuffer: imageBuffer, orientation: .leftMirrored, options: [:])
do {
try imageRequestHandler.perform([faceDetectionRequest])
} catch {
print(error.localizedDescription)
}
}
private func handleFaceDetectionObservations(
observations: [VNFaceObservation],
_ pixelBuffer: CVPixelBuffer,
_ sampleBuffer: CMSampleBuffer
) {
for observation in observations {
var faceRectConverted = self.previewLayer.layerRectConverted(fromMetadataOutputRect: observation.boundingBox)
let faceRectanglePath = CGPath(rect: faceRectConverted, transform: nil)
let faceLayer = CAShapeLayer()
faceLayer.path = faceRectanglePath
faceLayer.fillColor = UIColor.clear.cgColor
faceLayer.strokeColor = UIColor.yellow.cgColor
self.faceLayers.append(faceLayer)
self.view.layer.addSublayer(faceLayer)
if
let captureDevice = captureDevice,
let dockAccessory = dockAccessory
{
Task {
do {
try await trackWithDockKit(
observation.boundingBox,
dockAccessory,
pixelBuffer,
sampleBuffer
)
} catch {
print(error)
}
}
}
}
}
func trackWithDockKit(_ boundingBox: CGRect, _ dockAccessory: DockAccessory, _ pixelBuffer: CVPixelBuffer, _ cmSampelBuffer: CMSampleBuffer) async throws {
guard
let device = captureDevice
else {
fatalError("Kamera nicht verfügbar")
}
let size = CGSize(width: CVPixelBufferGetWidth(pixelBuffer), height: CVPixelBufferGetHeight(pixelBuffer))
let cameraInfo = DockAccessory.CameraInformation(
captureDevice: device.deviceType,
cameraPosition: device.position,
orientation: .corrected,
cameraIntrinsics: nil,
referenceDimensions: size
)
let observation = DockAccessory.Observation(
identifier: 0,
type: .object,
rect: boundingBox
)
let observations = [observation]
try await dockAccessory.track(observations, cameraInformation: cameraInfo)
}
}
Topic:
Media Technologies
SubTopic:
Photos & Camera
Tags: