I'm making video recording iOS app. I need to add timestamp to recorded video thus I convert CMSampleBuffer to UIImage and add timestamp on in and convert it back to CMSampleBuffer. While process of converting UIImage to CMSample buffer, I need to convert UIImage to CIImage but this process fails. How can I properly convert UIImage to CIImage?
func appendVideo(from sampleBuffer: CMSampleBuffer) {
// - Guards
guard let videoInput = assetWriter?.inputs.first(where: { $0.mediaType == .video }) else {
print("video input not found")
return
}
guard videoInput.isReadyForMoreMediaData else {
print("video input not ready for more media data")
return
}
// - Timestamp
let sample: Sample = Sample(sampleBuffer: sampleBuffer)
guard let ciImage = generateCIImage(from: sampleBuffer) else {
print("CIImage creation from sampleBuffer failed")
return
}
let uiImage = UIImage(ciImage: ciImage)
guard let timestampAddedImage = self.addTimestamp(on: uiImage) else {
fatalError("should not reach here")
}
////////////////////////////////////
// THIS PROCESS FAILS
////////////////////////////////////
guard let timestampAddedCiImage = timestampAddedImage.ciImage else {
print("UIImage to CIImage conversion faield")
return
}
guard let timestampAddedCvpixelBuffer = generateCVPixelBuffer(from: timestampAddedCiImage) else {
print("CVPixelBuffer creation from CIImage failed")
return
}
guard let timestampAddedSampleBuffer = generateCMSampleBuffer(from: timestampAddedCvpixelBuffer, timingInfo: sample.timingInfo) else {
print("CMSampleBuffer creation from CVPixelBufer failed")
return
}
DispatchQueue.main.sync { [weak self] in
self?.compositeImageView.image = timestampAddedImage
}
print("append video")
videoInput.append(timestampAddedSampleBuffer)
}
func addTimestamp(on image: UIImage) -> UIImage? {
let imageRect = CGRect(x: 0, y: 0, width: image.size.width, height: image.size.height)
UIGraphicsBeginImageContextWithOptions(image.size, true, 0.0)
image.draw(in: imageRect)
// Text Attributes
let textColor = UIColor.white
let textFont = UIFont.systemFont(ofSize: FontSize.sizeL, weight: .bold)
let textFontAttributes = [
NSAttributedString.Key.font: textFont,
NSAttributedString.Key.foregroundColor: textColor,
NSAttributedString.Key.backgroundColor: UIColor(hex: ColorConstants.black, alpha: 0.4)
]
// Timestamp to add to image
let formatter = DateFormatter()
formatter.dateFormat = "yyyy/MM/dd HH:mm:ss:SSS"
let timestamp: NSString = formatter.string(from: Date()) as NSString
let textRect = CGRect(x: 6.0, y: 6.0, width: image.size.width, height: 32)
timestamp.draw(in: textRect, withAttributes: textFontAttributes)
// New Image
let newImage = UIGraphicsGetImageFromCurrentImageContext();
UIGraphicsEndImageContext()
return newImage
}
// MARK: - CMSampleBuffer <-> Image Conversion
// ref: https://fromatom.hatenablog.com/entry/2019/10/28/172628
// CMSampleFubber > CIImage
func generateCIImage(from sampleBuffer: CMSampleBuffer) -> CIImage? {
guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else {
return nil
}
let ciImage = CIImage(cvPixelBuffer: pixelBuffer)
return ciImage
}
func generateCVPixelBuffer(from ciImage: CIImage) -> CVPixelBuffer? {
let attrs = [kCVPixelBufferCGImageCompatibilityKey: kCFBooleanTrue, kCVPixelBufferCGBitmapContextCompatibilityKey: kCFBooleanTrue] as CFDictionary
var pixelBuffer: CVPixelBuffer!
let status = CVPixelBufferCreate(kCFAllocatorDefault, Int(ciImage.extent.size.width), Int(ciImage.extent.size.height), kCVPixelFormatType_32BGRA, attrs, &pixelBuffer)
guard status == kCVReturnSuccess else {
print("CVPixelBufferCreateに失敗")
return nil
}
let ciContext = CIContext()
ciContext.render(ciImage, to: pixelBuffer, bounds: ciImage.extent, colorSpace: CGColorSpaceCreateDeviceRGB())
CVPixelBufferLockBaseAddress(pixelBuffer, CVPixelBufferLockFlags(rawValue: 0))
return pixelBuffer
}
func generateCMSampleBuffer(from cvPixelBuffer: CVPixelBuffer, timingInfo: CMSampleTimingInfo) -> CMSampleBuffer? {
var sampleBuffer: CMSampleBuffer?
var timimgInfo: CMSampleTimingInfo = timingInfo
var videoInfo: CMVideoFormatDescription!
CMVideoFormatDescriptionCreateForImageBuffer(allocator: nil, imageBuffer: cvPixelBuffer, formatDescriptionOut: &videoInfo)
CMSampleBufferCreateForImageBuffer(allocator: kCFAllocatorDefault,
imageBuffer: cvPixelBuffer,
dataReady: true,
makeDataReadyCallback: nil,
refcon: nil,
formatDescription: videoInfo,
sampleTiming: &timimgInfo,
sampleBufferOut: &sampleBuffer)
return sampleBuffer
}
private final class Sample {
let timingInfo: CMSampleTimingInfo
init(sampleBuffer: CMSampleBuffer) {
let presentationTimeStamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer)
let duration = CMSampleBufferGetDuration(sampleBuffer)
let decodeTimeStamp = CMSampleBufferGetDecodeTimeStamp(sampleBuffer)
timingInfo = CMSampleTimingInfo(duration: duration, presentationTimeStamp: presentationTimeStamp, decodeTimeStamp: decodeTimeStamp)
}
}
Please update your addTimestamp function like this and try again:
func addTimestamp(on image: UIImage) -> UIImage? {
let imageRect = CGRect(x: 0, y: 0, width: image.size.width, height: image.size.height)
UIGraphicsBeginImageContextWithOptions(image.size, true, 0.0)
image.draw(in: imageRect)
// Text Attributes
let textColor = UIColor.white
let textFont = UIFont.systemFont(ofSize: FontSize.sizeL, weight: .bold)
let textFontAttributes = [
NSAttributedString.Key.font: textFont,
NSAttributedString.Key.foregroundColor: textColor,
NSAttributedString.Key.backgroundColor: UIColor(hex: ColorConstants.black, alpha: 0.4)
]
// Timestamp to add to image
let formatter = DateFormatter()
formatter.dateFormat = "yyyy/MM/dd HH:mm:ss:SSS"
let timestamp: NSString = formatter.string(from: Date()) as NSString
let textRect = CGRect(x: 6.0, y: 6.0, width: image.size.width, height: 32)
timestamp.draw(in: textRect, withAttributes: textFontAttributes)
// New Image
guard let newImage = UIGraphicsGetImageFromCurrentImageContext(),
let cgImage = newImage.cgImage else { return nil }
UIGraphicsEndImageContext()
let ciImage = CIImage(cgImage: cgImage)
return UIImage(ciImage: ciImage)
}