Babylon React Native WebXR Demo issues

Hi there!

I’ve been trying to get the Babylon React Native WebXR demo found here(Babylon React Native Demo - YouTube, https://playground.babylonjs.com/#LQMZQ7#0) to work, but i’m having some issues. I basically copied most of the code, and merged with the Babylon React Native playground app.

First issue i’m having is related to earcut injection on the MeshBuilder.CreatePolygon function, i’ve imported earcut as described here: https://forum.babylonjs.com/t/how-to-inject-earcut-to-a-react-babylon-project/14905 but still, it’s throwing me this error :

[TypeError: this.bjsEarcut is not a function. (In 'this.bjsEarcut(this._epoints, this._eholes, 2)', 'this.bjsEarcut' is an instance of Object)]

at

 try {
                plane.mesh = MeshBuilder.CreatePolygon(
                  "plane",
                  { shape: plane.polygonDefinition },
                  scene,
                  earcut
                );

Second issue is related to the Device Source Manager observables not receiving updated conditional variables after initial call. I did :

useEffect(() => {
    createInputHandling();
  }, [xrSession]);

  const createInputHandling = useCallback(() => {
    async function inputListen() {
      //   var numInputs = 0;
      deviceSourceManager?.onDeviceConnectedObservable.add((device) => {
        // numInputs++;

        if (device.deviceType === DeviceType.Touch) {
          const touch: DeviceSource<DeviceType.Touch> =
            deviceSourceManager.getDeviceSource(
              device.deviceType,
              device.deviceSlot
            )!;
          touch.onInputChangedObservable.add((touchEvent) => {
         
            if (
              model &&
              xrSession &&
              placementIndicator?.isEnabled() &&
              !modelPlaced
            ) {
              placeModel();
              console.log("model placing input");
              //numInputs--;
            }

            if (
              model &&
              modelPlaced &&
              xrSession &&
              placementIndicator?.isEnabled() &&
              touchEvent.previousState !== null &&
              touchEvent.currentState !== null
            ) {
              console.log("update input");
              // Calculate the differential between two states.
              const diff = touchEvent.previousState - touchEvent.currentState;
              // Single input, do translation.
              //if (numInputs === 1) {
              if (touchEvent.inputIndex === PointerInput.Horizontal) {
                model.position.x -= diff / 1000;
              } else {
                model.position.z += diff / 750;
              }
              // }
              // Multi-input do rotation.
              //   if (
              //     numInputs === 2 &&
              //     touchEvent.inputIndex === PointerInput.Horizontal &&
              //     touchEvent.deviceSlot === 0
              //   ) {
              //     console.log("multi touch");
              //     model.rotate(Vector3.Up(), diff / 200);
              //   }
            }
          });
        } else if (device.deviceType === DeviceType.Mouse) {
          const mouse: DeviceSource<DeviceType.Mouse> =
            deviceSourceManager.getDeviceSource(
              device.deviceType,
              device.deviceSlot
            )!;
          mouse.onInputChangedObservable.add((mouseEvent) => {
            if (mouse.getInput(PointerInput.LeftClick)) {
              return;
            }
          });
        }
      });
    }
    inputListen();
  }, [xrSession, placementIndicator, modelPlaced]);

Initial touch places the model, set modelPlaced to true and disable placementIndicator, which should return false for the same condition on the next touch, however it’s always returning true. I’m not sure if i did some mistake here, or if i’m missing something out. Any inputs would be appreciated. Thanks

Full code:

import React, { useCallback, useEffect, useState } from "react";
import { View, ViewProps, Button, Image, Text } from "react-native";

import {
  EngineView,
  EngineViewCallbacks,
  useEngine,
} from "@babylonjs/react-native";

import {
  AbstractMesh,
  ArcRotateCamera,
  Camera,
  Color3,
  Color4,
  DeviceSource,
  DeviceSourceManager,
  DeviceType,
  HemisphericLight,
  Mesh,
  MeshBuilder,
  Nullable,
  PointerInput,
  Quaternion,
  Scene,
  SceneLoader,
  StandardMaterial,
  Texture,
  TransformNode,
  TubeBuilder,
  Vector3,
  WebXRFeatureName,
  WebXRHitTest,
  WebXRPlaneDetector,
  WebXRSessionManager,
  WebXRTrackingState,
} from "@babylonjs/core";
import "@babylonjs/loaders";
import { StyleProp, StyleSheet, ViewStyle } from "react-native";
// import { MathJaxSvg } from "react-native-mathjax-html-to-svg";
import Slider from "@react-native-community/slider";

//import { Slider, Text, Button, Image } from "react-native-elements";
import { SafeAreaView } from "react-native-safe-area-context";

import * as earcut from "earcut";

const EngineScreen = (props) => {
  const defaultScale = 1;
  const enableSnapshots = false;

  const engine = useEngine();
  const [toggleView, setToggleView] = useState(false);
  const [camera, setCamera] = useState<ArcRotateCamera | undefined>();
  const [rootNode, setRootNode] = useState<TransformNode | undefined>();
  const [scene, setScene] = useState<Scene>();
  const [xrSession, setXrSession] = useState<WebXRSessionManager>();
  const [scale, setScale] = useState<number>(defaultScale);
  const [snapshotData, setSnapshotData] = useState<string>();
  const [engineViewCallbacks, setEngineViewCallbacks] =
    useState<EngineViewCallbacks>();
  const [trackingState, setTrackingState] = useState<WebXRTrackingState>();

  const [placementIndicator, setPlacementIndicator] =
    useState<AbstractMesh | undefined>();
  const [model, setModel] = useState<AbstractMesh>();
  const [modelPlaced, setModelPlaced] = useState<boolean>(false);
  const [planeMat, setPlaneMat] = useState<any>();
  const [deviceSourceManager, setDeviceSourceManager] =
    useState<DeviceSourceManager>();

  useEffect(() => {
    async function init() {
      if (engine) {
        const scene = new Scene(engine);
        setScene(scene);
        scene.createDefaultCamera(true);
        (scene.activeCamera as ArcRotateCamera).beta -= Math.PI / 8;
        setCamera(scene.activeCamera! as ArcRotateCamera);
        //scene.createDefaultLight(true);
        //scene.clearColor = new Color4(0, 0, 0, 0);
        const light = new HemisphericLight(
          "light1",
          new Vector3(0, 5, 0),
          scene
        );
        light.diffuse = Color3.White();
        light.intensity = 1;
        light.specular = new Color3(0, 0, 0);

        // Create the placement indicator
        let placementIndicator = Mesh.CreateTorus(
          "placementIndicator",
          0.5,
          0.005,
          64
        );
        let indicatorMat = new StandardMaterial("noLight", scene);
        indicatorMat.disableLighting = true;
        indicatorMat.emissiveColor = Color3.White();
        placementIndicator.material = indicatorMat;
        placementIndicator.scaling = new Vector3(1, 0.01, 1);
        placementIndicator.setEnabled(false);
        setPlacementIndicator(placementIndicator);

        const rootNode = new TransformNode("Root Container", scene);
        setRootNode(rootNode);

        const transformContainer = new TransformNode(
          "Transform Container",
          scene
        );
        transformContainer.parent = rootNode;
        transformContainer.scaling.scaleInPlace(0.2);
        transformContainer.position.y -= 0.2;

        // import model
        SceneLoader.ImportMeshAsync(
          "",
          "https://raw.githubusercontent.com/KhronosGroup/glTF-Sample-Models/master/2.0/BoxAnimated/glTF-Binary/BoxAnimated.glb"
          //"https://raw.githubusercontent.com/KhronosGroup/glTF-Sample-Models/master/2.0/BrainStem/glTF/BrainStem.gltf"
        ).then((result) => {
          const mesh = result.meshes[0];

          mesh.position.y = 1;
          //mesh.lookAt(camera?.position);

          //mesh.scalingDeterminant = 0;
          mesh.parent = transformContainer;
          setModel(mesh);
          //camera?.setTarget(mesh);
        });

        const planeTexture = new Texture(
          "https://i.imgur.com/z7s3C5B.png",
          scene
        );
        planeTexture.hasAlpha = true;
        planeTexture.uScale = 3;
        planeTexture.vScale = 3;
        planeTexture.coordinatesMode = Texture.PROJECTION_MODE;

        const planeMat = new StandardMaterial("noLight", scene);
        planeMat.diffuseTexture = planeTexture;
        setPlaneMat(planeMat);

        const deviceSourceManager = new DeviceSourceManager(engine);
        setDeviceSourceManager(deviceSourceManager);
      }
    }
    init();
  }, [engine]);

  ////////////////
  const resetClick = () => {
    if (model && camera && scene && placementIndicator) {
      if (xrSession) {
        setModelPlaced(false);
        model.setEnabled(false);
        placementIndicator.setEnabled(true);
      } else {
        placementIndicator.setEnabled(false);
        // reset2D()
      }
    }
  };

  useEffect(() => {
    if (rootNode) {
      rootNode.scaling = new Vector3(scale, scale, scale);
    }
  }, [rootNode, scale]);

  useEffect(() => {
    createInputHandling();
  }, [xrSession]);

  const createInputHandling = useCallback(() => {
    async function inputListen() {
      //   var numInputs = 0;
      deviceSourceManager?.onDeviceConnectedObservable.add((device) => {
        // numInputs++;

        if (device.deviceType === DeviceType.Touch) {
          const touch: DeviceSource<DeviceType.Touch> =
            deviceSourceManager.getDeviceSource(
              device.deviceType,
              device.deviceSlot
            )!;
          touch.onInputChangedObservable.add((touchEvent) => {
            if (
              model &&
              xrSession &&
              placementIndicator?.isEnabled() &&
              !modelPlaced
            ) {
              placeModel();

              //placementIndicator?.setEnabled(false);
              console.log("model placing input");
              //numInputs--;
            }

            if (
              model &&
              modelPlaced &&
              xrSession &&
              placementIndicator?.isEnabled() &&
              touchEvent.previousState !== null &&
              touchEvent.currentState !== null
            ) {
              console.log("update input");
              // Calculate the differential between two states.
              const diff = touchEvent.previousState - touchEvent.currentState;
              // Single input, do translation.
              //if (numInputs === 1) {
              if (touchEvent.inputIndex === PointerInput.Horizontal) {
                model.position.x -= diff / 1000;
              } else {
                model.position.z += diff / 750;
              }
              // }
              // Multi-input do rotation.
              //   if (
              //     numInputs === 2 &&
              //     touchEvent.inputIndex === PointerInput.Horizontal &&
              //     touchEvent.deviceSlot === 0
              //   ) {
              //     console.log("multi touch");
              //     model.rotate(Vector3.Up(), diff / 200);
              //   }
            }
          });
        } else if (device.deviceType === DeviceType.Mouse) {
          const mouse: DeviceSource<DeviceType.Mouse> =
            deviceSourceManager.getDeviceSource(
              device.deviceType,
              device.deviceSlot
            )!;
          mouse.onInputChangedObservable.add((mouseEvent) => {
            if (mouse.getInput(PointerInput.LeftClick)) {
              return;
            }
          });
        }
      });
    }
    inputListen();
  }, [xrSession, placementIndicator, modelPlaced]);

  const reset2D = () => {
    if (model && scene && camera) {
      model.setEnabled(true);
      model.position = camera.position.add(
        camera.getForwardRay().direction.scale(scale * 1)
      );
      placementIndicator?.setEnabled(false);
      setModelPlaced(false);
      //model.scalingDeterminant = 0;
      //camera.setTarget(model)
    }
  };

  const placeModel = () => {
    console.log("placeModel");
    if (
      xrSession &&
      placementIndicator?.isEnabled() &&
      scene &&
      model &&
      !modelPlaced
    ) {
      setModelPlaced(true);
      model.rotationQuaternion = Quaternion.Identity();

      model.position = placementIndicator.position.clone();
      placementIndicator?.setEnabled(false);
      model.setEnabled(true);

      //model.scalingDeterminant = 0;
    }
  };

  const trackingStateToString = (
    trackingState: WebXRTrackingState | undefined
  ): string => {
    return trackingState === undefined ? "" : WebXRTrackingState[trackingState];
  };

  const onToggleXr = useCallback(() => {
    (async () => {
      if (xrSession) {
        reset2D();
        await xrSession.exitXRAsync();
      } else {
        if (rootNode !== undefined && scene !== undefined) {
          const xr = await scene.createDefaultXRExperienceAsync({
            disableDefaultUI: true,
            disableTeleportation: true,
          });
          const session = await xr.baseExperience.enterXRAsync(
            "immersive-ar",
            "unbounded",
            xr.renderTarget
          );

          setModelPlaced(false);
          model?.setEnabled(false);

          setXrSession(session);
          session.onXRSessionEnded.add(() => {
            setXrSession(undefined);
            setTrackingState(undefined);
          });

          setTrackingState(xr.baseExperience.camera.trackingState);
          xr.baseExperience.camera.onTrackingStateChanged.add(
            (newTrackingState) => {
              setTrackingState(newTrackingState);
            }
          );
          // Set up the hit test.
          const xrHitTestModule =
            xr.baseExperience.featuresManager.enableFeature(
              WebXRFeatureName.HIT_TEST,
              "latest",
              {
                offsetRay: {
                  origin: { x: 0, y: 0, z: 0 },
                  direction: { x: 0, y: 0, z: -1 },
                },
              }
            ) as WebXRHitTest;

          // Do some plane shtuff.
          const xrPlanes = xr.baseExperience.featuresManager.enableFeature(
            WebXRFeatureName.PLANE_DETECTION,
            "latest"
          ) as WebXRPlaneDetector;
          console.log("Enabled plane detection.");
          const planes: any[] = [];

          xrPlanes.onPlaneAddedObservable.add((webXRPlane) => {
            if (scene) {
              console.log("Plane added.");
              let plane: any = webXRPlane;
              webXRPlane.polygonDefinition.push(
                webXRPlane.polygonDefinition[0]
              );
              try {
                plane.mesh = MeshBuilder.CreatePolygon(
                  "plane",
                  { shape: plane.polygonDefinition },
                  scene,
                  earcut
                );
                let tubeMesh: Mesh = TubeBuilder.CreateTube(
                  "tube",
                  {
                    path: plane.polygonDefinition,
                    radius: 0.005,
                    sideOrientation: Mesh.FRONTSIDE,
                    updatable: true,
                  },
                  scene
                );
                tubeMesh.setParent(plane.mesh);
                planes[plane.id] = plane.mesh;
                plane.mesh.material = planeMat;

                plane.mesh.rotationQuaternion = new Quaternion();
                plane.transformationMatrix.decompose(
                  plane.mesh.scaling,
                  plane.mesh.rotationQuaternion,
                  plane.mesh.position
                );
              } catch (ex) {
                console.error(ex);
              }
            }
          });

          xrPlanes.onPlaneUpdatedObservable.add((webXRPlane) => {
            console.log("Plane updated.");
            let plane: any = webXRPlane;
            if (plane.mesh) {
              plane.mesh.dispose(false, false);
            }

            const some = plane.polygonDefinition.some((p: any) => !p);
            if (some) {
              return;
            }

            plane.polygonDefinition.push(plane.polygonDefinition[0]);
            try {
              plane.mesh = MeshBuilder.CreatePolygon(
                "plane",
                { shape: plane.polygonDefinition },
                scene,
                earcut
              );
              let tubeMesh: Mesh = TubeBuilder.CreateTube(
                "tube",
                {
                  path: plane.polygonDefinition,
                  radius: 0.005,
                  sideOrientation: Mesh.FRONTSIDE,
                  updatable: true,
                },
                scene
              );
              tubeMesh.setParent(plane.mesh);
              planes[plane.id] = plane.mesh;
              plane.mesh.material = planeMat;
              plane.mesh.rotationQuaternion = new Quaternion();
              plane.transformationMatrix.decompose(
                plane.mesh.scaling,
                plane.mesh.rotationQuaternion,
                plane.mesh.position
              );
              plane.mesh.receiveShadows = true;
            } catch (ex) {
              console.error(ex);
            }
          });

          xrPlanes.onPlaneRemovedObservable.add((webXRPlane) => {
            console.log("Plane removed.");
            let plane: any = webXRPlane;
            if (plane && planes[plane.id]) {
              planes[plane.id].dispose();
            }
          });

          xrHitTestModule.onHitTestResultObservable.add((results) => {
            if (results.length) {
              if (!modelPlaced) {
                placementIndicator?.setEnabled(true);
              } else {
                placementIndicator?.setEnabled(false);
              }

              if (placementIndicator) {
                placementIndicator.position = results[0].position;
              }
            }
          });
        }
      }
    })();
  }, [
    rootNode,
    scene,
    xrSession
  ]);

  const onInitialized = useCallback(
    async (engineViewCallbacks: EngineViewCallbacks) => {
      setEngineViewCallbacks(engineViewCallbacks);
    },
    [engine]
  );

  const onSnapshot = useCallback(async () => {
    if (engineViewCallbacks) {
      setSnapshotData(
        "data:image/jpeg;base64," + (await engineViewCallbacks.takeSnapshot())
      );
    }
  }, [engineViewCallbacks]);

  return (
    <>
      <View style={props.style}>
        <Button
          title="Toggle EngineView"
          onPress={() => {
            setToggleView(!toggleView);
          }}
        />
        <Button
          title={xrSession ? "Stop XR" : "Start XR"}
          onPress={() => {
            //desabilitar o model assim que inicia a xrsession, no usecallback não funciona
            if (!xrSession) {
              model?.setEnabled(false);
            }
            onToggleXr();
          }}
        />

        {!toggleView && (
          <View style={{ flex: 1 }}>
            {enableSnapshots && (
              <View style={{ flex: 1 }}>
                <Button title={"Take Snapshot"} onPress={onSnapshot} />
                <Image style={{ flex: 1 }} source={{ uri: snapshotData }} />
              </View>
            )}
            <EngineView
              style={props.style}
              camera={camera}
              onInitialized={onInitialized}
            />
            <Slider
              style={{
                position: "absolute",
                minHeight: 50,
                margin: 10,
                left: 0,
                right: 0,
                bottom: 0,
              }}
              minimumValue={0.2}
              maximumValue={2}
              step={0.01}
              value={defaultScale}
              onValueChange={setScale}
            />
            <Text
              style={{
                fontSize: 12,
                color: "yellow",
                position: "absolute",
                margin: 10,
              }}
            >
              {trackingStateToString(trackingState)}
            </Text>
          </View>
        )}
        {toggleView && (
          <View
            style={{ flex: 1, justifyContent: "center", alignItems: "center" }}
          >
            <Text style={{ fontSize: 24 }}>EngineView has been removed.</Text>
            <Text style={{ fontSize: 12 }}>
              Render loop stopped, but engine is still alive.
            </Text>
          </View>
        )}
      </View>
    </>
  );
};

const BabylonXrTest = ({ route, navigation }) => {
  const [toggleScreen, setToggleScreen] = useState(false);
  //const { topic } = route.params;

  return (
    <>
      <SafeAreaView style={{ flex: 1, backgroundColor: "white" }}>
        {!toggleScreen && <EngineScreen style={{ flex: 1 }} />}
        {toggleScreen && (
          <View
            style={{ flex: 1, justifyContent: "center", alignItems: "center" }}
          >
            <Text style={{ fontSize: 24 }}>EngineScreen has been removed.</Text>
            <Text style={{ fontSize: 12 }}>
              Engine has been disposed, and will be recreated.
            </Text>
          </View>
        )}
        <Button
          title="Toggle EngineScreen"
          onPress={() => {
            setToggleScreen(!toggleScreen);
          }}
        />
      </SafeAreaView>
    </>
  );
};

export default BabylonXrTest;

const styles = StyleSheet.create({
  engineView: {
    backgroundColor: "transparent",
  },
});

Adding @brianzinn

How did you import earcut. This works for me in non-native:

import * as Earcut from 'earcut';

// then you pass in Earcut to the earcutInjection parameter in the creation method.

I have this in my package.json:

"earcut": "^2.2.2",

It’s the 6th parameter and I think you have it as the 4th parameter:
MeshBuilder | Babylon.js Documentation (babylonjs.com)

edit: you imported earcut OK. the * export can have any name.

1 Like