Detect gesture in immersive space VisionOs SwiftUI

Question

I have an immersive space that I want to add a gesture recognizer to. I want the ability to detect a pinch / tap gesture anywhere in the immersive space.

ImmersiveSpace(id: "FlappyImmersiveSpace") {     
    FlappySpace() 
}

I tried this, it didn't work.

ImmersiveSpace(id: "FlappyImmersiveSpace") {
    FlappySpace()
        .gesture(
            TapGesture()
                .onEnded({ _ in
                    print("TAPPED")
                })
        )
}

Neither does this.

struct FlappySpace: View {

    var body: some View {
        RealityView { content in
            // Logic
        }
        .gesture(
            TapGesture()
                .onEnded({ _ in
                    print("TAPPED")
                })
        )
    }
}

Does any one know how to detect gestures in an immersive space, the gesture can not be specific to an entity in the space, but any where in the entire space. Would specifically like to detect a tap gesture (in VisionOS, this is a pinch).

Doralin

SpatialTapGesture – Canner

Answer 1

You need to call a Spatial Tap Gesture

Like this

        .gesture(
            SpatialTapGesture()
                .targetedToEntity(entity)
                .onEnded { value in
                    print("you touch me")

                }
        )

Here is a full example with a mesh cube you can touch it with your finger to animate and jump up , using double tap it would jump up more

import SwiftUI
import RealityKit


struct Cube: View {
    @State var z: Float = 0.0
    @State private var subs: [EventSubscription] = []
    @State private var contentEntity = Entity()
    @State var cube1 = ModelEntity()



    var body: some View {
        RealityView { content, attachments in
            content.add(setupContentEntity())
            
            cube1 = addCube(name: "Cube1", posision: SIMD3(x: 1, y: 1, z: -2), color: .red)
            
            if let attachment = attachments.entity(for: "cube1_label") {
                attachment.position = [0, -0.35, 0]
                cube1.addChild(attachment)
            }
            
        } attachments: {
            Attachment(id: "cube1_label") {
                Text("Cube1")
                    .font(.system(size: 48))
            }
        }
        .gesture(
            SpatialTapGesture()
                .targetedToEntity(cube1)
                .onEnded { value in
                    print(value)
                   playAnimation(entity: cube1)
                }
        )
    }
    
    func setupContentEntity() -> Entity {
        return contentEntity
    }

    func getTargetEntity(name: String) -> Entity? {
        return contentEntity.children.first { $0.name == name}
    }

    func addCube(name: String, posision: SIMD3<Float>, color: UIColor) -> ModelEntity {
        let entity = ModelEntity(
            mesh: .generateBox(size: 0.5, cornerRadius: 0),
            materials: [SimpleMaterial(color: color, isMetallic: false)],
            collisionShape: .generateBox(size: SIMD3<Float>(repeating: 0.5)),
            mass: 0.0
        )

        entity.name = name
        entity.position = posision
        entity.components.set(InputTargetComponent(allowedInputTypes: .indirect))
        entity.components.set(CollisionComponent(shapes: [ShapeResource.generateBox(size: SIMD3<Float>(repeating: 0.5))], isStatic: true))
        entity.components.set(HoverEffectComponent())

        contentEntity.addChild(entity)
        
        return entity
    }

    func playAnimation(entity: Entity) {
        let goUp = FromToByAnimation<Transform>(
            name: "goUp",
            from: .init(scale: .init(repeating: 1), translation: entity.position),
            to: .init(scale: .init(repeating: 1), translation: entity.position + .init(x: 0, y: 0.4, z: 0)),
            duration: 0.2,
            timing: .easeOut,
            bindTarget: .transform
        )

        let pause = FromToByAnimation<Transform>(
            name: "pause",
            from: .init(scale: .init(repeating: 1), translation: entity.position + .init(x: 0, y: 0.4, z: 0)),
            to: .init(scale: .init(repeating: 1), translation: entity.position + .init(x: 0, y: 0.4, z: 0)),
            duration: 0.1,
            bindTarget: .transform
        )

        let goDown = FromToByAnimation<Transform>(
            name: "goDown",
            from: .init(scale: .init(repeating: 1), translation: entity.position + .init(x: 0, y: 0.4, z: 0)),
            to: .init(scale: .init(repeating: 1), translation: entity.position),
            duration: 0.2,
            timing: .easeOut,
            bindTarget: .transform
        )

        let goUpAnimation = try! AnimationResource
            .generate(with: goUp)

        let pauseAnimation = try! AnimationResource
            .generate(with: pause)

        let goDownAnimation = try! AnimationResource
            .generate(with: goDown)

        let animation = try! AnimationResource.sequence(with: [goUpAnimation, pauseAnimation, goDownAnimation])

        entity.playAnimation(animation, transitionDuration: 0.5)
    }

}

Karrikarrie

That is really not answering to the question, which was "I want the ability to detect a pinch / tap gesture anywhere in the immersive space". Your response gives (a nice) example of how to target entities with tap gestures. I am also atm looking for a way to reliably generate drag / tap events on a ImmersiveSpace. One use case for a "tap anywhere" could be to bring up video players controls that had disappeared automatically, or in case of a "drag anywhere", to move one's entire 3D scene around. – Prospector

@Matti, How about sending this gesture to the entire content would this help? also you can put a transparent texture sphere as a child of your head position and that way when you touch anywhere the sphere is reacted? – Karrikarrie

yes I've been pondering about adding a huge spherical collision shape to an invisible object but it would seem the collision shapes are not detected from the inside and I'm not sure if their surfaces can be flipped. – Prospector

@Matti, You have to give it a try, in fact visionOS is still new and missing a lot so we have to invent ideas , like im stuck on the stage of creating a dashboard for the game but then the window context cannot change its position so I need to make it 3d icons and I dunno how to add interactive text as a texture on this 3d model but also I need to give it a try. good luck with that – Karrikarrie

Same issue as @Matti. Found lots on getting entities to target/tap, but doesn't work when you're INSIDE the sphere or for any taps that aren't on an entity. Any anyone found any workarounds? – Greenfinch

@Matti, I think for the sphere to work you need to flip the normals so it would be seen from inside, but that gonna make another challenge because making it transparent from the visible side does not look good for the eye , its kinda putting a glass on your eye. and if this not work then you need to divide the mesh of the sphere into 8 sections or more then apply the collision mesh on each and also it need to be flipped to be seen from inside. after all I guess Apple have a reason for not allowing the tap anywhere in visionOS. – Karrikarrie

Did you find a solution to this? I was wondering if we need to create our own global gestures by constantly tracking fingers. Seems a bit overkill, but it might work. – Croaker

@TobyEvetts It seems global gestures are nearly impossible unless you won't need to click anything else except one – Karrikarrie

Yes, that's what worried me. I did try adding a large hemisphere in the background and making it virtually transparent. The problem is that that now makes interaction with anything in the foreground much more hit and miss as the system is constantly checking everywhere for interactions. – Croaker

Hey @Karrikarrie I might have a similar problem, can you help me - #79038340 – Atory

Answer 2

Start by making an anchor to the head of the user and then add a child to it which is A 3d Sphere invisible from inside and since you are inside it you won't see it then you can add a gesture on it to be clickable and it would be touched from anywhere

import SwiftUI
import RealityKit


struct MyView: View {

@State var myHead: Entity = {
    let headAnchor = AnchorEntity(.head)
    headAnchor.position = [0, -0.15, -0.4]
    return headAnchor
}()

@State var sphere: ModelEntity = {
    let dashboardEntity = ModelEntity(mesh: .generateSphere(radius: 100), materials: [])
    dashboardEntity.generateCollisionShapes(recursive: false)
    dashboardEntity.components.set(InputTargetComponent())
    return dashboardEntity
}()

@State var box: ModelEntity = {
    let entity = ModelEntity(mesh: .generateBox(size: [0.1, 0.1, 0.1]), materials: [SimpleMaterial()])
    entity.position = [0, 1.6, -0.7]

    return entity
}()



@State var clicked = false

var clickedMaterial = SimpleMaterial(color: .red, isMetallic: false)

var unclickedMaterial = SimpleMaterial(color: .green, isMetallic: false)

var body: some View {
    RealityView { content in
        content.add(box)

        content.add(myHead)
        
        myHead.addChild(sphere)

    }
    update: { content in
        // Update the dashboard entity's material when the value of `clicked` changes.
        box.model?.materials = clicked ? [clickedMaterial] : [unclickedMaterial]
    }
    .gesture(
        TapGesture()
            .targetedToEntity(sphere)
            .onEnded({ value in
                // Toggle `clicked` when the dashboard entity is tapped.
                clicked.toggle()
            })
        )
}
}

Karrikarrie