swiftuivirtual-realityrealitykitios18realityview

How to control Stereo VR RealityView with touch or Game Controller in iOS 18 or MacOS 15


I have two identical views showing stereo VR view calling same structure twice in HStack to allow a one RealityView to be shown twice as a VR solution but I still failed to control both views at same time, I have a drag option but it work on each view separately not together at same time, and also I have double-click option to stop animation and restart it also it works separately not in sync

Also when i try to move the entity using a Game Controller it moves only at one view not both and this is a weird action cause it is the same variable and it should act together in sync.

My Question is how to control both views together with either touch or game controller to be able to move both entities on both sides together in sync

here is the code for the one controlled by the touch

import SwiftUI
import RealityKit

struct ContentView : View {
  var body: some View {
    HStack {
        MainView()
        MainView()
    }
    .background(.black)
  }
}


struct MainView : View {
  @State var anchor1 = AnchorEntity()
  @State var rotateTime: Timer!
  @State var rotateCounter: Float = 0.0

  @State var animateMode = false

  @State var position: UnitPoint = .zero

  let ratio: Float = 0.005

  var body: some View {
    RealityView { rvc in
        let item = ModelEntity(mesh: .generateBox(size: 0.5), materials: [SimpleMaterial()])
        anchor1.addChild(item)
        anchor1.orientation = .init(angle: .pi/4, axis:[0,1,1])
        rvc.add(anchor1)
    }
    .onAppear {
        DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) {
            rotateTime?.invalidate()
            rotateTime = Timer.scheduledTimer(
                withTimeInterval: 0.01, repeats: true, block: { _ in
                    rotate(anchor1)
                }
            )
        }
    }
    .gesture(dragThis)
    .gesture(doubleClickThis)
}

func rotate(_ anchor: AnchorEntity) {
    rotateCounter += 1
    
    if rotateCounter >= 360 { rotateCounter = 0.0 }
    
    anchor.orientation = .init(angle: (rotateCounter * (.pi/180)), axis: [0,1,0])
}

var dragThis: some Gesture {
    DragGesture(minimumDistance: 15, coordinateSpace: .global)
        .onChanged { value in
            print(value.translation)
            print(value.location)
            
            anchor1.position.x = Float(value.translation.width + position.x) * ratio
            anchor1.position.y = Float(value.translation.height + position.y) * -ratio
            
        }
        .onEnded{value in
            position.x += value.translation.width
            position.y += value.translation.height
        }
}

var doubleClickThis: some Gesture {
    TapGesture(count: 2)
        .onEnded { value in
            animateMode.toggle()
            
            if animateMode {
                rotateTime?.invalidate()
                rotateTime = Timer.scheduledTimer(
                    withTimeInterval: 0.01, repeats: true, block: { _ in
                        rotate(anchor1)
                        //rotate(anchor2)
                    }
                )
            }
            else
            {
                rotateTime?.invalidate()
            }
        }
  }
}

Since in VR we cannot touch the screen so here is the code with a control by both touch and using a Game Controller

import SwiftUI
import RealityKit
import GameController

struct ContentView : View {
var body: some View {
    HStack {
        MainView()
        MainView()
    }
    .background(.black)
}
}


struct MainView : View {
@State var anchor1 = AnchorEntity()
@State var rotateTime: Timer!
@State var rotateCounter: Float = 0.0

@State var animateMode = false

@State var position: UnitPoint = .zero

@State var gameController: GCController!

@State private var stickTime: Timer!

@State var SRDelayer = false

let ratio: Float = 0.005

var body: some View {
    RealityView { rvc in
        let item = ModelEntity(mesh: .generateBox(size: 0.5), materials: [SimpleMaterial()])
        anchor1.addChild(item)
        anchor1.orientation = .init(angle: .pi/4, axis:[0,1,1])
        rvc.add(anchor1)
    }
    .onAppear {
        setupGameController()
        
        DispatchQueue.main.asyncAfter(deadline: .now() + 2.0) {
            rotateTime?.invalidate()
            rotateTime = Timer.scheduledTimer(
                withTimeInterval: 0.01, repeats: true, block: { _ in
                    rotate(anchor1)
                }
            )
        }
    }
    .gesture(dragThis)
    .gesture(doubleClickThis)
}

func rotate(_ anchor: AnchorEntity) {
    rotateCounter += 1
    
    if rotateCounter >= 360 { rotateCounter = 0.0 }
    
    anchor.orientation = .init(angle: (rotateCounter * (.pi/180)), axis: [0,1,0])
}

var dragThis: some Gesture {
    DragGesture(minimumDistance: 15, coordinateSpace: .global)
        .onChanged { value in
            print(value.translation)
            print(value.location)
            
            anchor1.position.x = Float(value.translation.width + position.x) * ratio
            anchor1.position.y = Float(value.translation.height + position.y) * -ratio
            
        }
        .onEnded{value in
            position.x += value.translation.width
            position.y += value.translation.height
        }
}

var doubleClickThis: some Gesture {
    TapGesture(count: 2)
        .onEnded { value in
            animateMode.toggle()
            playStop()
        }
}

func playStop() {
    if animateMode {
        rotateTime?.invalidate()
        rotateTime = Timer.scheduledTimer(
            withTimeInterval: 0.01, repeats: true, block: { _ in
                rotate(anchor1)
                //rotate(anchor2)
            }
        )
    }
    else
    {
        rotateTime?.invalidate()
    }
}

//MARK: - GameController

func setupGameController() {
    
    NotificationCenter.default.addObserver(forName: NSNotification.Name.GCControllerDidBecomeCurrent, object: nil, queue: nil, using: didConnectController)
    
    NotificationCenter.default.addObserver(forName: NSNotification.Name.GCControllerDidStopBeingCurrent, object: nil, queue: nil, using: didDisconnectController)
    
    GCController.startWirelessControllerDiscovery{}
    
    guard let controller = GCController.controllers().first
    else
    {
        return
    }
    registerGameController(controller)
    
}

func didConnectController(_ notification: Notification) {
    
    gameController = notification.object as? GCController
    
    print("Game Controller ◦ gameControllerIsConnected \(gameController.productCategory)")
    
    unregisterGameController()
    
    registerGameController(gameController)
}

func didDisconnectController(_ notification: Notification) {
    
    gameController = notification.object as? GCController
    print("◦ disgameControllerIsConnected \(gameController.productCategory)")
    
    unregisterGameController()
}

func registerGameController(_ gameController: GCController) {
    
    self.gameController = gameController // important for refresh screen
    
    gameController.extendedGamepad?.dpad.left.pressedChangedHandler = { (button, value, pressed) in self.button("Left", pressed) }
    gameController.extendedGamepad?.dpad.up.pressedChangedHandler = { (button, value, pressed) in self.button("Up", pressed) }
    gameController.extendedGamepad?.dpad.right.pressedChangedHandler = { (button, value, pressed) in self.button("Right", pressed) }
    gameController.extendedGamepad?.dpad.down.pressedChangedHandler = { (button, value, pressed) in self.button("Down", pressed) }
    gameController.extendedGamepad?.buttonX.pressedChangedHandler = { (button, value, pressed) in self.button("Square", pressed) }
    gameController.extendedGamepad?.buttonY.pressedChangedHandler = { (button, value, pressed) in self.button("Triangle", pressed) }
    gameController.extendedGamepad?.buttonB.pressedChangedHandler = { (button, value, pressed) in self.button("Circle", pressed) }
    gameController.extendedGamepad?.buttonA.pressedChangedHandler = { (button, value, pressed) in self.button("Cross", pressed) }
    gameController.extendedGamepad?.buttonOptions?.pressedChangedHandler = { (button, value, pressed) in self.button("Options", pressed) }
    gameController.extendedGamepad?.buttonMenu.pressedChangedHandler = { (button, value, pressed) in self.button("Menu", pressed) }
    gameController.extendedGamepad?.leftThumbstickButton?.pressedChangedHandler = { (button, value, pressed) in self.button("LeftThumbClick", pressed) }
    gameController.extendedGamepad?.rightThumbstickButton?.pressedChangedHandler = { (button, value, pressed) in self.button("RightThumbClick", pressed) }
    gameController.extendedGamepad?.leftShoulder.pressedChangedHandler = { (button, value, pressed) in self.button("LeftShoulder", pressed) }
    gameController.extendedGamepad?.rightShoulder.pressedChangedHandler = { (button, value, pressed) in self.button("RightShoulder", pressed) }
    gameController.extendedGamepad?.leftTrigger.pressedChangedHandler = { (button, value, pressed) in self.button("LeftTriggerClick", pressed)}
    gameController.extendedGamepad?.rightTrigger.pressedChangedHandler = { (button, value, pressed) in self.button("RightTriggerClick", pressed) }
    gameController.extendedGamepad?.leftTrigger.valueChangedHandler = { (button, value, pressed) in self.triggerL("LeftTriggerValue", value) }
    gameController.extendedGamepad?.rightTrigger.valueChangedHandler = { (button, value, pressed) in self.triggerR("RightTriggerValue", value) }
    gameController.extendedGamepad?.leftThumbstick.valueChangedHandler = { (button, xvalue, yvalue) in self.sticks("LeftThumbStick", xvalue, yvalue) }
    gameController.extendedGamepad?.rightThumbstick.valueChangedHandler = { (button, xvalue, yvalue) in self.sticks("RightThumbStick", xvalue, yvalue) }
    
}

func unregisterGameController() {
    
}

func button(_ button: String, _ pressed: Bool) {
    
    if button == "Cross"
    {
        if SRDelayer == false
        {
            SRDelayer = true
            
            animateMode.toggle()
            playStop()
        }
        
        DispatchQueue.main.asyncAfter(deadline: DispatchTime.now() + 1.0)
        {
            self.SRDelayer = false
        }
    }
    
    
    if button == "Up"
    {
        anchor1.position.y += 0.02
    }
    
    if button == "Down"
    {
        anchor1.position.y -= 0.02
    }
    
    if button == "Right"
    {
        anchor1.position.x += 0.02
    }
    
    if button == "Left"
    {
        anchor1.position.x -= 0.02
    }
}

func triggerR(_ button: String, _ value: Float) {

}

func triggerL(_ button: String, _ value: Float) {
}

func sticks(_ button: String, _ xvalue: Float, _ yvalue: Float) {
    if button == "RightThumbStick" || button == "LeftThumbStick"
    {
        if yvalue > 0.5 || yvalue < -0.5  || xvalue > 0.5 || xvalue < -0.5
        {
            if stickTime == nil // if its not running
            {
                stickTime?.invalidate()
                
                stickTime = Timer.scheduledTimer(withTimeInterval: 0.01, repeats: true, block: { _ in
                    pollInputSticks()
                })
            }
        }
        else
        {
            stickTime?.invalidate()
            stickTime = nil
        }
    }
    else
    {
        stickTime?.invalidate()
        stickTime = nil
    }
}

func pollInputSticks() {
    if let gamePadRight = gameController.extendedGamepad?.rightThumbstick
    {
        if gamePadRight.yAxis.value > 0.5 // Up
        {
            anchor1.position.y += 0.002
        }
        else if gamePadRight.yAxis.value < -0.5 // Down
        {
            anchor1.position.y -= 0.002
        }
        
        if gamePadRight.xAxis.value > 0.5 // Right
        {
            anchor1.position.x += 0.002
        }
        else if gamePadRight.xAxis.value < -0.5 // Left
        {
            anchor1.position.x -= 0.002
        }
    }
    
    if let gamePadLeft = gameController.extendedGamepad?.leftThumbstick
    {
        if gamePadLeft.yAxis.value > 0.5 // Up
        {
            anchor1.position.z -= 0.002
        }
        else if gamePadLeft.yAxis.value < -0.5 // Down
        {
            anchor1.position.z += 0.002
        }
        
        if gamePadLeft.xAxis.value > 0.5 // Right
        {
            anchor1.position.x += 0.002
        }
        else if gamePadLeft.xAxis.value < -0.5 // Left
        {
            anchor1.position.x -= 0.002
        }
    }
}

}

here is the results photos from real iPhone11 with iOS18

enter image description here With No Control

enter image description here Using Touches

enter image description here Using GameController


Solution

  • The main idea of ​​how to make an in-sync anchors' XY-offset in both SwiftUI views (when you are moving just a single object with a drag gesture) is that you need to capture the state's change for the pose object in the body of .onChange(of:) method and then pass the corresponding XY values ​​to the anchor1 and anchor2. To control a rotation, create a new @State property similar to the pose pty.

    enter image description here

    Here's my code. Run it on iOS device only (iOS Simulator isn't working properly).

    import SwiftUI
    import RealityKit
    
    struct ContentView : View {
        let anchor1 = AnchorEntity()
        let anchor2 = AnchorEntity()
        @State var pose: SIMD3<Float> = .zero
    
        var body: some View {
            HStack {
                VStack {
                    Text(anchor1.position.description)
                        .font(.headline)
                        .padding(15)
                    MainView(anchor: anchor1, pose: $pose)
                        .onChange(of: pose) { (_, new) in
                            anchor2.position.x = Float(new.x)
                            anchor2.position.y = Float(new.y)
                        }
                }
                VStack {
                    Text(anchor2.position.description)
                        .font(.headline)
                        .padding(15)
                    MainView(anchor: anchor2, pose: $pose)
                        .onChange(of: pose) { (_, new) in
                            anchor1.position.x = Float(new.x)
                            anchor1.position.y = Float(new.y)
                        }
                }
            }
            .backgroundStyle(.black)
        }
    }
    

    struct MainView : View {
        let anchor: AnchorEntity
        @Binding var pose: SIMD3<Float>
        @State var position: UnitPoint = .zero
        @State var rotateTime: Timer!
        @State var rotateCounter: Float = 0.0
        @State var animateMode = false
        let ratio: Float = 0.005
        
        var body: some View {
            RealityView { rvc in
                let item = ModelEntity(mesh: .generateBox(size: 0.5),
                                  materials: [SimpleMaterial()])
                anchor.addChild(item)
                anchor.orientation = .init(angle: .pi/4, axis:[0,1,1])
                rvc.add(anchor)
            }
            .onAppear {
                DispatchQueue.main.asyncAfter(deadline: .now() + 1.0) {
                    rotateTime?.invalidate()
                    rotateTime = Timer.scheduledTimer(
                        withTimeInterval: 0.01, repeats: true) { _ in
                            rotate(anchor)
                        }
                }
            }
            .gesture(
                DragGesture(minimumDistance: 15, coordinateSpace: .global)
                    .onChanged {
                        anchor.position.x = Float($0.translation.width + position.x) * ratio
                        anchor.position.y = Float($0.translation.height + position.y) * -ratio
                        
                        pose.x = anchor.position.x
                        pose.y = anchor.position.y
                    }
                    .onEnded {
                        position.x += $0.translation.width
                        position.y += $0.translation.height
                        print(pose)
                    }
            )
            .gesture(
                TapGesture(count: 2)
                    .onEnded { _ in
                        animateMode.toggle()
                    
                        if animateMode {
                            rotateTime?.invalidate()
                            rotateTime = Timer.scheduledTimer(
                                withTimeInterval: 0.01, repeats: true) { _ in
                                    rotate(anchor)
                            }
                        } else {
                            rotateTime?.invalidate()
                        }
                    }
            )
        }
        
        func rotate(_ anchor: AnchorEntity) {
            rotateCounter += 1
            if rotateCounter >= 360 { rotateCounter = 0.0 }
            anchor.orientation = .init(angle: (rotateCounter * (.pi/180)),
                                        axis: [0,1,0])
        }
    }