Augmented Reality with ARKit and RealityKit

// FieldNotes/AR/NotesMapPreview.swift (macOS window content)
#if os(macOS)
import FieldNotesServices
import RealityKit
import SwiftUI

struct NotesMapPreview: View {
    @Environment(NoteStore.self) private var store

    var body: some View {
        RealityView { content in
            content.camera = .virtual
            let root = Entity()
            for (note, position) in NotesMapLayout.positions(for: store.notes) {
                root.addChild(NotePinFactory.makePin(for: note, at: position))
            }
            content.add(root)
        }
        .realityViewCameraControls(.orbit)
    }
}

#Preview {
    NotesMapPreview()
        .environment(NoteStore(notes: [/* sample notes with locations */]))
}
#endif

// FieldNotes/AR/FieldNotesARView.swift
#if os(iOS)
import ARKit
import FieldNotesModel
import FieldNotesServices
import RealityKit
import SwiftUI

struct FieldNotesARView: View {
    @Environment(NoteStore.self) private var store
    @State private var trackingSession = SpatialTrackingSession()
    @State private var selectedNote: Note?
    @State private var trackingProblem: String?

    var body: some View {
        RealityView { content in
            content.camera = .spatialTracking

            // Wait for a real horizontal table at least 30 cm x 30 cm, then place the pins on it.
            let table = AnchorEntity(.plane(.horizontal, classification: .table,
                                            minimumBounds: [0.3, 0.3]),
                                     trackingMode: .once)
            for (note, position) in NotesMapLayout.positions(for: store.notes, tableSize: 0.3) {
                table.addChild(NotePinFactory.makePin(for: note, at: position))
            }
            content.add(table)
        }
        .gesture(
            SpatialTapGesture()
                .targetedToAnyEntity()
                .onEnded { value in
                    guard let pin = value.entity.components[NotePinComponent.self] else { return }
                    selectedNote = store.notes.first { $0.id == pin.noteID }
                }
        )
        .sheet(item: $selectedNote) { note in
            NoteDetail(note: note)
        }
        .overlay(alignment: .top) {
            if let trackingProblem { Text(trackingProblem).padding().background(.regularMaterial) }
        }
        .task {
            guard ARWorldTrackingConfiguration.isSupported else {
                trackingProblem = "This device doesn't support world tracking."
                return
            }
            let configuration = SpatialTrackingSession.Configuration(tracking: [.plane, .camera])
            if let unavailable = await trackingSession.run(configuration) {
                trackingProblem = "Some tracking is unavailable: \(unavailable)"
            }
        }
    }
}
#endif

// ContentView (iOS excerpt): presenting it, with the store injected explicitly.
// .fullScreenCover(isPresented: $showsARPins) {
//     FieldNotesARView().environment(store)
// }

// FieldNotes/FieldNotesApp.swift (visionOS excerpt, updated)
#if os(visionOS)
ImmersiveSpace(id: "map") {
    NotesMapSpace()
        .environment(store)          // NotesMapSpace reads @Environment(NoteStore.self)
}
#endif

// FieldNotes/AR/NotesMapTracking.swift
#if os(visionOS)
import ARKit
import Observation
import RealityKit

@Observable @MainActor
final class NotesMapTracking {
    private(set) var tableTransform: simd_float4x4?
    let meshRoot = Entity()                 // collision meshes of the room
    let fingertip = ModelEntity(mesh: .generateSphere(radius: 0.006),
                                materials: [UnlitMaterial(color: .white)])

    private let session = ARKitSession()
    private let planes = PlaneDetectionProvider(alignments: [.horizontal])
    private let meshes = SceneReconstructionProvider()
    private let hands = HandTrackingProvider()
    private var meshEntities: [UUID: Entity] = [:]

    func run() async {
        var providers: [any DataProvider] = []
        if PlaneDetectionProvider.isSupported { providers.append(planes) }
        if SceneReconstructionProvider.isSupported { providers.append(meshes) }
        if HandTrackingProvider.isSupported { providers.append(hands) }
        do {
            try await session.run(providers)          // shows the permission prompts if needed
        } catch {
            return
        }
        await withTaskGroup(of: Void.self) { group in
            group.addTask { await self.processPlanes() }
            group.addTask { await self.processMeshes() }
            group.addTask { await self.processHands() }
        }
    }

    private func processPlanes() async {
        for await update in planes.anchorUpdates where update.anchor.classification == .table {
            if update.event != .removed, tableTransform == nil {
                tableTransform = update.anchor.originFromAnchorTransform    // first table wins
            }
        }
    }

    private func processMeshes() async {
        for await update in meshes.anchorUpdates {
            let anchor = update.anchor
            switch update.event {
            case .added, .updated:
                guard let shape = try? await ShapeResource.generateStaticMesh(from: anchor) else { continue }
                let entity = meshEntities[anchor.id] ?? Entity()
                entity.components.set(CollisionComponent(shapes: [shape], isStatic: true))
                entity.setTransformMatrix(anchor.originFromAnchorTransform, relativeTo: nil)
                if meshEntities[anchor.id] == nil {
                    meshEntities[anchor.id] = entity
                    meshRoot.addChild(entity)
                }
            case .removed:
                meshEntities.removeValue(forKey: anchor.id)?.removeFromParent()
            }
        }
    }

    private func processHands() async {
        for await update in hands.anchorUpdates {
            let hand = update.anchor
            guard hand.isTracked, hand.chirality == .right,
                  let tip = hand.handSkeleton?.joint(.indexFingerTip), tip.isTracked else { continue }
            fingertip.setTransformMatrix(hand.originFromAnchorTransform * tip.anchorFromJointTransform,
                                         relativeTo: nil)
        }
    }
}
#endif

// FieldNotes/Views/NotesMapSpace.swift
#if os(visionOS)
import FieldNotesModel
import FieldNotesServices
import RealityKit
import RealityKitContent
import SwiftUI

struct NotesMapSpace: View {
    @Environment(NoteStore.self) private var store
    @State private var tracking = NotesMapTracking()
    @State private var selectedNote: Note?

    var body: some View {
        RealityView { content in
            let map = Entity()
            map.name = "NotesMap"
            if let terrain = try? await Entity(named: "FieldSiteTerrain", in: realityKitContentBundle) {
                map.addChild(terrain)                                   // authored in Reality Composer Pro
            }
            for (note, position) in NotesMapLayout.positions(for: store.notes) {
                map.addChild(NotePinFactory.makePin(for: note, at: position))
            }
            map.isEnabled = false                                        // hidden until a table is found
            content.add(map)
            content.add(tracking.meshRoot)
            content.add(tracking.fingertip)
        } update: { content in
            guard let table = tracking.tableTransform,
                  let map = content.entities.first(where: { $0.name == "NotesMap" }) else { return }
            map.transform = Transform(matrix: table)
            map.isEnabled = true
        }
        .gesture(
            SpatialTapGesture()
                .targetedToAnyEntity()
                .onEnded { value in
                    guard let pin = value.entity.components[NotePinComponent.self] else { return }
                    selectedNote = store.notes.first { $0.id == pin.noteID }   // shown in a window, see below
                }
        )
        .task { await tracking.run() }
    }
}
#endif

let worldTracking = WorldTrackingProvider()
// after session.run([worldTracking, ...]):
try await worldTracking.addAnchor(WorldAnchor(originFromAnchorTransform: table))

import RealityKit
import RealityKitContent

let terrain = try await Entity(named: "FieldSiteTerrain", in: realityKitContentBundle)

// NotesMapSpace, inside make: three versions of the terrain from Reality Composer Pro
if #available(visionOS 27, iOS 27, macOS 27, *),
   let high = try? await Entity(named: "FieldSiteTerrain", in: realityKitContentBundle),
   let medium = try? await Entity(named: "FieldSiteTerrainMedium", in: realityKitContentBundle),
   let low = try? await Entity(named: "FieldSiteTerrainLow", in: realityKitContentBundle) {
    let terrain = Entity()
    LevelOfDetailComponent.addByCameraDistance(to: terrain, levels: [
        (entities: [high], maxDistance: 1.0),        // metres
        (entities: [medium], maxDistance: 3.0),
        (entities: [low], maxDistance: .infinity),
    ])
    map.addChild(terrain)
}