Using replaykit or any other framework to capture whole screen and not just the app with script

I am trying to build an app that streams Iphone screen to PC over the wifi network. I am able to capture app's screen and receive it on my PC but if the app goes in background (I exit using home button), the stream still shows the app screen. I want to show it the current visible UI on Iphone. This is the complete script.

import Foundation
import AVFoundation
import ReplayKit
import UIKit

class ScreenRecorder: NSObject, ObservableObject, StreamDelegate {
    private var recorder = RPScreenRecorder.shared()
    private var outputStream: OutputStream?
    private var backgroundTask: UIBackgroundTaskIdentifier = .invalid

    override init() {
        super.init()
    }

    func startCapturing() {
        setupStream()
        registerBackgroundTask()
        recorder.isMicrophoneEnabled = false
        recorder.startCapture(handler: { [weak self] (sampleBuffer, bufferType, error) in
            if let error = error {
                print("Error capturing sample buffer: \(error.localizedDescription)")
                return
            }
            if bufferType == .video {
                self?.processSampleBuffer(sampleBuffer: sampleBuffer)
            }
        }) { error in
            if let error = error {
                print("Error starting capture: \(error.localizedDescription)")
            } else {
                print("Capture started successfully")
            }
        }
    }

    func stopCapturing() {
        recorder.stopCapture { [weak self] error in
            if let error = error {
                print("Error stopping capture: \(error.localizedDescription)")
            }
            self?.outputStream?.close()
            self?.outputStream = nil
        }
        endBackgroundTask()
    }

    private func processSampleBuffer(sampleBuffer: CMSampleBuffer) {
        guard let outputStream = outputStream else {
            print("Output stream is nil")
            return
        }

        guard CMSampleBufferDataIsReady(sampleBuffer) else {
            print("Sample buffer data is not ready")
            return
        }

        if let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) {
        
            let ciImage = CIImage(cvImageBuffer: imageBuffer)
            let context = CIContext()
            
        
            if let jpegData = context.jpegRepresentation(of: ciImage, colorSpace: CGColorSpaceCreateDeviceRGB(), options: [:]) {
                var frameSize = UInt32(jpegData.count)
                var sizeData = Data(bytes: &frameSize, count: MemoryLayout<UInt32>.size)
                
                // Send the frame size
                sizeData.withUnsafeBytes { (rawBufferPointer: UnsafeRawBufferPointer) in
                    if let baseAddress = rawBufferPointer.baseAddress {
                        let bytesWritten = outputStream.write(baseAddress.assumingMemoryBound(to: UInt8.self), maxLength: sizeData.count)
                        if bytesWritten <= 0 {
                            print("Error writing frame size to output stream: \(outputStream.streamError?.localizedDescription ?? "Unknown error")")
                            return
                        } else {
                            print("Wrote frame size \(frameSize) to output stream")
                        }
                    }
                }
                
            
                jpegData.withUnsafeBytes { (rawBufferPointer: UnsafeRawBufferPointer) in
                    if let baseAddress = rawBufferPointer.baseAddress {
                        let bytesWritten = outputStream.write(baseAddress.assumingMemoryBound(to: UInt8.self), maxLength: jpegData.count)
                        if bytesWritten <= 0 {
                            print("Error writing frame data to output stream: \(outputStream.streamError?.localizedDescription ?? "Unknown error")")
                        } else {
                            print("Wrote \(bytesWritten) bytes of sample buffer data to the output stream")
                        }
                    }
                }
            } else {
                print("Error converting CIImage to JPEG data")
            }
        } else {
            print("Failed to get image buffer")
        }
    }


    func setupStream() {
        var readStream: Unmanaged<CFReadStream>?
        var writeStream: Unmanaged<CFWriteStream>?

        let ipAddress = "192.168.x.xx" 
        let port: UInt32 = 5001
        print("Attempting to connect to \(ipAddress) on port \(port)")

        CFStreamCreatePairWithSocketToHost(kCFAllocatorDefault, ipAddress as CFString, port, &readStream, &writeStream)

        guard let outStream = writeStream?.takeRetainedValue() else {
            print("Failed to create write stream")
            return
        }
        outputStream = outStream
        outputStream?.delegate = self
        outputStream?.schedule(in: .current, forMode: .default)
        outputStream?.open()

        print("Stream setup complete, attempting to open output stream")
    }

    func stream(_ aStream: Stream, handle eventCode: Stream.Event) {
        switch eventCode {
        case .openCompleted:
            print("Stream opened successfully")
        case .hasBytesAvailable:
            print("Stream has bytes available")
        case .hasSpaceAvailable:
            print("Stream has space available")
        case .errorOccurred:
            if let error = aStream.streamError {
                print("Stream error occurred: \(error)")
            }
        case .endEncountered:
            print("Stream end encountered")
            aStream.close()
            aStream.remove(from: .current, forMode: .default)
        default:
            print("Unknown stream event")
        }
    }

    private func registerBackgroundTask() {
        backgroundTask = UIApplication.shared.beginBackgroundTask { [weak self] in
            self?.endBackgroundTask()
        }
        assert(backgroundTask != .invalid)
    }

    private func endBackgroundTask() {
        if backgroundTask != .invalid {
            UIApplication.shared.endBackgroundTask(backgroundTask)
            backgroundTask = .invalid
        }
    }
}

It only shows this much on my mac screen. Please help!. I am not able to figure out a way to stream my complete screen. Is it possible?

iOS version - 18 Xcode version - 16 Macos version - sequoia 15

Edit:

I tried using RPBroadcastHandler as suggested in comment so I,

Added New target in my App (Broadcast upload extension) and got a new target scaffolded in my App.
This is what the project structure looks like now.
The updated SampleHandler.swift looks like this

import ReplayKit
import VideoToolbox

class SampleHandler: RPBroadcastSampleHandler {
    var outputStream: OutputStream?

    override func broadcastStarted(withSetupInfo setupInfo: [String : NSObject]?) {
        setupStream()
    }
    
    override func broadcastPaused() { }
    
    override func broadcastResumed() { }
    
    override func broadcastFinished() {
        outputStream?.close()
    }
    
    func setupStream() {
        var writeStream: Unmanaged<CFWriteStream>?
        let ipAddress = "192.168.1.38"
        let port: UInt32 = 5000
        
        CFStreamCreatePairWithSocketToHost(kCFAllocatorDefault, ipAddress as CFString, port, nil, &writeStream)
        
        outputStream = writeStream?.takeRetainedValue()
        outputStream?.schedule(in: .current, forMode: .default)
        outputStream?.open()
    }
    
    override func processSampleBuffer(_ sampleBuffer: CMSampleBuffer, with sampleBufferType: RPSampleBufferType) {
        guard let outputStream = outputStream, sampleBufferType == .video else { return }
        
        guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
        let ciImage = CIImage(cvImageBuffer: imageBuffer)
        let context = CIContext()
        
        if let jpegData = context.jpegRepresentation(of: ciImage, colorSpace: CGColorSpaceCreateDeviceRGB(), options: [:]) {
            let _ = jpegData.withUnsafeBytes { outputStream.write($0.bindMemory(to: UInt8.self).baseAddress!, maxLength: jpegData.count) }
        }
    }
}

I left the other files unchanged and "Build and Run".
Xcode prompts me to select an app to run, I choose "ScreenStreamer" As it is what includes the extension.
Once the app is installed, it shows this dialog.
Error details -

The specified capability is not supported by this device.
Domain: com.apple.dt.CoreDeviceError
Code: 1001
User Info: {
    DVTErrorCreationDateKey = "2024-08-02 12:18:47 +0000";
    IDERunOperationFailingWorker = IDEDebugXPCCoreDeviceWorker;
    ServiceName = "com.apple.instruments.dtservicehub";
}
--

Event Metadata: com.apple.dt.IDERunOperationWorkerFinished : {
    "device_identifier" = "00008020-000848683A84002E";
    "device_isCoreDevice" = 1;
    "device_model" = "iPhone11,6";
    "device_osBuild" = "18.0 (22A5316j)";
    "device_platform" = "com.apple.platform.iphoneos";
    "device_thinningType" = "iPhone11,6";
    "dvt_coredevice_version" = "397.16";
    "dvt_coresimulator_version" = 986;
    "dvt_mobiledevice_version" = "1759.0.5";
    "launchSession_schemeCommand" = Run;
    "launchSession_state" = 1;
    "launchSession_targetArch" = arm64;
    "operation_duration_ms" = 3252;
    "operation_errorCode" = 1001;
    "operation_errorDomain" = "com.apple.dt.CoreDeviceError";
    "operation_errorWorker" = IDEDebugXPCCoreDeviceWorker;
    "operation_name" = IDERunOperationWorkerGroup;
    "param_debugger_attachToExtensions" = 1;
    "param_debugger_attachToXPC" = 1;
    "param_debugger_type" = 1;
    "param_destination_isProxy" = 0;
    "param_destination_platform" = "com.apple.platform.iphoneos";
    "param_diag_113575882_enable" = 0;
    "param_diag_MainThreadChecker_stopOnIssue" = 0;
    "param_diag_MallocStackLogging_enableDuringAttach" = 0;
    "param_diag_MallocStackLogging_enableForXPC" = 1;
    "param_diag_allowLocationSimulation" = 1;
    "param_diag_checker_tpc_enable" = 1;
    "param_diag_gpu_frameCapture_enable" = 0;
    "param_diag_gpu_shaderValidation_enable" = 0;
    "param_diag_gpu_validation_enable" = 0;
    "param_diag_memoryGraphOnResourceException" = 0;
    "param_diag_mtc_enable" = 1;
    "param_diag_queueDebugging_enable" = 1;
    "param_diag_runtimeProfile_generate" = 0;
    "param_diag_sanitizer_asan_enable" = 0;
    "param_diag_sanitizer_tsan_enable" = 0;
    "param_diag_sanitizer_tsan_stopOnIssue" = 0;
    "param_diag_sanitizer_ubsan_enable" = 0;
    "param_diag_sanitizer_ubsan_stopOnIssue" = 0;
    "param_diag_showNonLocalizedStrings" = 0;
    "param_diag_viewDebugging_enabled" = 1;
    "param_diag_viewDebugging_insertDylibOnLaunch" = 1;
    "param_install_style" = 2;
    "param_launcher_UID" = 2;
    "param_launcher_allowDeviceSensorReplayData" = 0;
    "param_launcher_kind" = 0;
    "param_launcher_style" = 0;
    "param_launcher_substyle" = 2;
    "param_runnable_appExtensionHostRunMode" = 0;
    "param_runnable_productType" = "com.apple.product-type.app-extension";
    "param_structuredConsoleMode" = 1;
    "param_testing_launchedForTesting" = 0;
    "param_testing_suppressSimulatorApp" = 0;
    "param_testing_usingCLI" = 0;
    "sdk_canonicalName" = "iphoneos18.0";
    "sdk_osVersion" = "18.0";
    "sdk_variant" = iphoneos;
}
--


System Information

macOS Version 15.0 (Build 24A5298h)
Xcode 16.0 (23047) (Build 16A5211f)
Timestamp: 2024-08-02T17:48:47+05:30

I am not sure what is causing this error.

Solution

Thank you @iSpain17 @GoksuBayy for suggestions. Here's what worked for me. I set up the broadcast extension like usual and created a picker button to launch the Broadcast Extension picker.

ViewController.swift

import UIKit
import ReplayKit

class ViewController: UIViewController {
    override func viewDidLoad() {
        super.viewDidLoad()
        view.addSubview(pickerButton)
        pickerButton.translatesAutoresizingMaskIntoConstraints = false
        NSLayoutConstraint.activate([
            pickerButton.centerXAnchor.constraint(equalTo: view.centerXAnchor),
            pickerButton.centerYAnchor.constraint(equalTo: view.centerYAnchor),
            pickerButton.widthAnchor.constraint(equalToConstant: 44),
            pickerButton.heightAnchor.constraint(equalToConstant: 44)
        ])
        view.bringSubviewToFront(pickerButton)
    }
    let pickerButton: RPSystemBroadcastPickerView = {
        let picker = RPSystemBroadcastPickerView(frame: CGRect(origin: .zero, size: CGSize(width: 44, height: 44)))
        picker.preferredExtension = "com.MH.screen-sharing-example.screen-share-extension"
        return picker
    }()
   
}

Then for processing Video buffer, this implementation worked for me.

SampleHandler.swift

//
//  SampleHandler.swift
//  screen-share-extension
//
//  Created by Prayas Gautam on 05/08/24.
//

import UIKit
import ReplayKit
import Network
import Foundation


class SampleHandler: RPBroadcastSampleHandler {
    
    var connection: NWConnection?
    var lastSendTime: TimeInterval = 0
    let sendInterval: TimeInterval = 1.0 / 10.0 // 30 FPS

    override init() {
        super.init()
        setupConnection()
        NotificationCenter.default.addObserver(self, selector: #selector(appDidEnterBackground), name: UIApplication.didEnterBackgroundNotification, object: nil)
        NotificationCenter.default.addObserver(self, selector: #selector(appWillEnterForeground), name: UIApplication.willEnterForegroundNotification, object: nil)
    }
    
    deinit {
        NotificationCenter.default.removeObserver(self)
    }
    
    func setupConnection() {
        let host = NWEndpoint.Host("192.168.x.xx")
        let port = NWEndpoint.Port(integerLiteral: 8080)
        connection = NWConnection(host: host, port: port, using: .udp)
        connection?.start(queue: .main)
    }
    
    func processVideoSampleBuffer(_ sampleBuffer: CMSampleBuffer) {
        guard let connection = connection else { return }
               
               // Convert sample buffer to data
       guard let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return }
       let ciImage = CIImage(cvImageBuffer: imageBuffer)
       let context = CIContext()
       guard let cgImage = context.createCGImage(ciImage, from: ciImage.extent) else { return }
       
       // Resize the image
       let uiImage = UIImage(cgImage: cgImage)
       let size = CGSize(width: uiImage.size.width / 2, height: uiImage.size.height / 2) // Reduce resolution by half
       UIGraphicsBeginImageContext(size)
       uiImage.draw(in: CGRect(origin: .zero, size: size))
       let resizedImage = UIGraphicsGetImageFromCurrentImageContext()
       UIGraphicsEndImageContext()
       
       guard let jpegData = resizedImage?.jpegData(compressionQuality: 0.2) else { return } // Adjust compression quality
       
       // Send data over UDP
       connection.send(content: jpegData, completion: .contentProcessed({ sendError in
           if let sendError = sendError {
               NSLog("Failed to send data: \(sendError)")
           } else {
               NSLog("Data sent successfully")
           }
       }))
    }
    
    @objc func appDidEnterBackground() {
        // Pause streaming or reduce frequency
        NSLog("App entered background")
    }
    
    @objc func appWillEnterForeground() {
        // Resume streaming
        NSLog("App will enter foreground")
    }
    override func broadcastStarted(withSetupInfo setupInfo: [String : NSObject]?) {
        // User has requested to start the broadcast. Setup info from the UI extension can be supplied but optional. 
    }
    
    override func broadcastPaused() {
        // User has requested to pause the broadcast. Samples will stop being delivered.
    }
    
    override func broadcastResumed() {
        // User has requested to resume the broadcast. Samples delivery will resume.
    }
    
    override func broadcastFinished() {
        // User has requested to finish the broadcast.
    }
    
    override func processSampleBuffer(_ sampleBuffer: CMSampleBuffer, with sampleBufferType: RPSampleBufferType) {
        switch sampleBufferType {
        case RPSampleBufferType.video:
            let currentTime = CACurrentMediaTime()
            if currentTime - lastSendTime >= sendInterval {
                lastSendTime = currentTime
                processVideoSampleBuffer(sampleBuffer)
            }
            break
        case RPSampleBufferType.audioApp:
            // Handle audio sample buffer for app audio
            break
        case RPSampleBufferType.audioMic:
            // Handle audio sample buffer for mic audio
            break
        @unknown default:
            // Handle other sample buffer types
            fatalError("Unknown type of sample buffer")
        }
    }
}

Python script to receive the stream on PC.

import socket
import cv2
import numpy as np

# Set up socket
UDP_IP = "0.0.0.0"
UDP_PORT = 8080
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind((UDP_IP, UDP_PORT))

while True:
    data, addr = sock.recvfrom(65536)  

    np_data = np.frombuffer(data, dtype=np.uint8)
    frame = cv2.imdecode(np_data, 1)
    
    if frame is not None:
        cv2.imshow('Video Stream', frame)
        
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break

# Release resources
sock.close()
cv2.destroyAllWindows()