Trying to export time ranges from a composition using AVAssetExportSession - "Decode timestamp is earlier than previous sample's decode timestamp"

Question

Created Dec ’21

Replies 0

Boosts 0

Participants 1

Modifying guidance given in an answer on AVFoundation + Vision trajectory detection, I'm instead saving time ranges of frames that have a specific ML label from my custom action classifier:

private lazy var detectHumanBodyPoseRequest: VNDetectHumanBodyPoseRequest = {
    let detectHumanBodyPoseRequest = VNDetectHumanBodyPoseRequest(completionHandler: completionHandler)
    return detectHumanBodyPoseRequest
}()

var timeRangesOfInterest: [Int : CMTimeRange] = [:]

private func readingAndWritingDidFinish(assetReaderWriter: AVAssetReaderWriter,
                                        asset
                                        completionHandler: @escaping FinishHandler) {
    if isCancelled {
        completionHandler(.success(.cancelled))
        return
    }

    // Handle any error during processing of the video.
    guard sampleTransferError == nil else {
        assetReaderWriter.cancel()
        completionHandler(.failure(sampleTransferError!))
        return
    }

    // Evaluate the result reading the samples.
    let result = assetReaderWriter.readingCompleted()
    if case .failure = result {
        completionHandler(result)
        return
    }

    /*
     Finish writing, and asynchronously evaluate the results from writing
     the samples.
    */
    assetReaderWriter.writingCompleted { result in
        self.exportVideoTimeRanges(timeRanges: self.timeRangesOfInterest.map { $0.value }) { result in
            completionHandler(result)
        }
    }
}

func exportVideoTimeRanges(timeRanges: [CMTimeRange], completion: @escaping (Result<OperationStatus, Error>) -> Void) {
    let inputVideoTrack = self.asset.tracks(withMediaType: .video).first!
    
    let composition = AVMutableComposition()
    
    let compositionTrack = composition.addMutableTrack(withMediaType: .video, preferredTrackID: kCMPersistentTrackID_Invalid)!
    
    var insertionPoint: CMTime = .zero
    for timeRange in timeRanges {
        try! compositionTrack.insertTimeRange(timeRange, of: inputVideoTrack, at: insertionPoint)
        insertionPoint = insertionPoint + timeRange.duration
    }
    
    let exportSession = AVAssetExportSession(asset: composition, presetName: AVAssetExportPresetHighestQuality)!
    try? FileManager.default.removeItem(at: self.outputURL)
    exportSession.outputURL = self.outputURL
    exportSession.outputFileType = .mov
    
    exportSession.exportAsynchronously {
        var result: Result<OperationStatus, Error>
        
        switch exportSession.status {
        case .completed:
            result = .success(.completed)
        case .cancelled:
            result = .success(.cancelled)
        case .failed:
            // The `error` property is non-nil in the `.failed` status.
            result = .failure(exportSession.error!)
        default:
            fatalError("Unexpected terminal export session status: \(exportSession.status).")
        }
        
        print("export finished: \(exportSession.status.rawValue) - \(exportSession.error)")
        
        completion(result)
    }
}

This worked fine with results vended from Apple's trajectory detection, but using my custom action classifier TennisActionClassifier (Core ML model exported from Create ML), I get the console error getSubtractiveDecodeDuration signalled err=-16364 (kMediaSampleTimingGeneratorError_InvalidTimeStamp) (Decode timestamp is earlier than previous sample's decode timestamp.) at MediaSampleTimingGenerator.c:180. Why might this be?

Boost