Hi there!
I’ve been trying to get the Babylon React Native WebXR demo found here(Babylon React Native Demo - YouTube, https://playground.babylonjs.com/#LQMZQ7#0) to work, but i’m having some issues. I basically copied most of the code, and merged with the Babylon React Native playground app.
First issue i’m having is related to earcut injection on the MeshBuilder.CreatePolygon function, i’ve imported earcut as described here: https://forum.babylonjs.com/t/how-to-inject-earcut-to-a-react-babylon-project/14905 but still, it’s throwing me this error :
[TypeError: this.bjsEarcut is not a function. (In 'this.bjsEarcut(this._epoints, this._eholes, 2)', 'this.bjsEarcut' is an instance of Object)]
at
try {
plane.mesh = MeshBuilder.CreatePolygon(
"plane",
{ shape: plane.polygonDefinition },
scene,
earcut
);
Second issue is related to the Device Source Manager observables not receiving updated conditional variables after initial call. I did :
useEffect(() => {
createInputHandling();
}, [xrSession]);
const createInputHandling = useCallback(() => {
async function inputListen() {
// var numInputs = 0;
deviceSourceManager?.onDeviceConnectedObservable.add((device) => {
// numInputs++;
if (device.deviceType === DeviceType.Touch) {
const touch: DeviceSource<DeviceType.Touch> =
deviceSourceManager.getDeviceSource(
device.deviceType,
device.deviceSlot
)!;
touch.onInputChangedObservable.add((touchEvent) => {
if (
model &&
xrSession &&
placementIndicator?.isEnabled() &&
!modelPlaced
) {
placeModel();
console.log("model placing input");
//numInputs--;
}
if (
model &&
modelPlaced &&
xrSession &&
placementIndicator?.isEnabled() &&
touchEvent.previousState !== null &&
touchEvent.currentState !== null
) {
console.log("update input");
// Calculate the differential between two states.
const diff = touchEvent.previousState - touchEvent.currentState;
// Single input, do translation.
//if (numInputs === 1) {
if (touchEvent.inputIndex === PointerInput.Horizontal) {
model.position.x -= diff / 1000;
} else {
model.position.z += diff / 750;
}
// }
// Multi-input do rotation.
// if (
// numInputs === 2 &&
// touchEvent.inputIndex === PointerInput.Horizontal &&
// touchEvent.deviceSlot === 0
// ) {
// console.log("multi touch");
// model.rotate(Vector3.Up(), diff / 200);
// }
}
});
} else if (device.deviceType === DeviceType.Mouse) {
const mouse: DeviceSource<DeviceType.Mouse> =
deviceSourceManager.getDeviceSource(
device.deviceType,
device.deviceSlot
)!;
mouse.onInputChangedObservable.add((mouseEvent) => {
if (mouse.getInput(PointerInput.LeftClick)) {
return;
}
});
}
});
}
inputListen();
}, [xrSession, placementIndicator, modelPlaced]);
Initial touch places the model, set modelPlaced to true and disable placementIndicator, which should return false for the same condition on the next touch, however it’s always returning true. I’m not sure if i did some mistake here, or if i’m missing something out. Any inputs would be appreciated. Thanks
Full code:
import React, { useCallback, useEffect, useState } from "react";
import { View, ViewProps, Button, Image, Text } from "react-native";
import {
EngineView,
EngineViewCallbacks,
useEngine,
} from "@babylonjs/react-native";
import {
AbstractMesh,
ArcRotateCamera,
Camera,
Color3,
Color4,
DeviceSource,
DeviceSourceManager,
DeviceType,
HemisphericLight,
Mesh,
MeshBuilder,
Nullable,
PointerInput,
Quaternion,
Scene,
SceneLoader,
StandardMaterial,
Texture,
TransformNode,
TubeBuilder,
Vector3,
WebXRFeatureName,
WebXRHitTest,
WebXRPlaneDetector,
WebXRSessionManager,
WebXRTrackingState,
} from "@babylonjs/core";
import "@babylonjs/loaders";
import { StyleProp, StyleSheet, ViewStyle } from "react-native";
// import { MathJaxSvg } from "react-native-mathjax-html-to-svg";
import Slider from "@react-native-community/slider";
//import { Slider, Text, Button, Image } from "react-native-elements";
import { SafeAreaView } from "react-native-safe-area-context";
import * as earcut from "earcut";
const EngineScreen = (props) => {
const defaultScale = 1;
const enableSnapshots = false;
const engine = useEngine();
const [toggleView, setToggleView] = useState(false);
const [camera, setCamera] = useState<ArcRotateCamera | undefined>();
const [rootNode, setRootNode] = useState<TransformNode | undefined>();
const [scene, setScene] = useState<Scene>();
const [xrSession, setXrSession] = useState<WebXRSessionManager>();
const [scale, setScale] = useState<number>(defaultScale);
const [snapshotData, setSnapshotData] = useState<string>();
const [engineViewCallbacks, setEngineViewCallbacks] =
useState<EngineViewCallbacks>();
const [trackingState, setTrackingState] = useState<WebXRTrackingState>();
const [placementIndicator, setPlacementIndicator] =
useState<AbstractMesh | undefined>();
const [model, setModel] = useState<AbstractMesh>();
const [modelPlaced, setModelPlaced] = useState<boolean>(false);
const [planeMat, setPlaneMat] = useState<any>();
const [deviceSourceManager, setDeviceSourceManager] =
useState<DeviceSourceManager>();
useEffect(() => {
async function init() {
if (engine) {
const scene = new Scene(engine);
setScene(scene);
scene.createDefaultCamera(true);
(scene.activeCamera as ArcRotateCamera).beta -= Math.PI / 8;
setCamera(scene.activeCamera! as ArcRotateCamera);
//scene.createDefaultLight(true);
//scene.clearColor = new Color4(0, 0, 0, 0);
const light = new HemisphericLight(
"light1",
new Vector3(0, 5, 0),
scene
);
light.diffuse = Color3.White();
light.intensity = 1;
light.specular = new Color3(0, 0, 0);
// Create the placement indicator
let placementIndicator = Mesh.CreateTorus(
"placementIndicator",
0.5,
0.005,
64
);
let indicatorMat = new StandardMaterial("noLight", scene);
indicatorMat.disableLighting = true;
indicatorMat.emissiveColor = Color3.White();
placementIndicator.material = indicatorMat;
placementIndicator.scaling = new Vector3(1, 0.01, 1);
placementIndicator.setEnabled(false);
setPlacementIndicator(placementIndicator);
const rootNode = new TransformNode("Root Container", scene);
setRootNode(rootNode);
const transformContainer = new TransformNode(
"Transform Container",
scene
);
transformContainer.parent = rootNode;
transformContainer.scaling.scaleInPlace(0.2);
transformContainer.position.y -= 0.2;
// import model
SceneLoader.ImportMeshAsync(
"",
"https://raw.githubusercontent.com/KhronosGroup/glTF-Sample-Models/master/2.0/BoxAnimated/glTF-Binary/BoxAnimated.glb"
//"https://raw.githubusercontent.com/KhronosGroup/glTF-Sample-Models/master/2.0/BrainStem/glTF/BrainStem.gltf"
).then((result) => {
const mesh = result.meshes[0];
mesh.position.y = 1;
//mesh.lookAt(camera?.position);
//mesh.scalingDeterminant = 0;
mesh.parent = transformContainer;
setModel(mesh);
//camera?.setTarget(mesh);
});
const planeTexture = new Texture(
"https://i.imgur.com/z7s3C5B.png",
scene
);
planeTexture.hasAlpha = true;
planeTexture.uScale = 3;
planeTexture.vScale = 3;
planeTexture.coordinatesMode = Texture.PROJECTION_MODE;
const planeMat = new StandardMaterial("noLight", scene);
planeMat.diffuseTexture = planeTexture;
setPlaneMat(planeMat);
const deviceSourceManager = new DeviceSourceManager(engine);
setDeviceSourceManager(deviceSourceManager);
}
}
init();
}, [engine]);
////////////////
const resetClick = () => {
if (model && camera && scene && placementIndicator) {
if (xrSession) {
setModelPlaced(false);
model.setEnabled(false);
placementIndicator.setEnabled(true);
} else {
placementIndicator.setEnabled(false);
// reset2D()
}
}
};
useEffect(() => {
if (rootNode) {
rootNode.scaling = new Vector3(scale, scale, scale);
}
}, [rootNode, scale]);
useEffect(() => {
createInputHandling();
}, [xrSession]);
const createInputHandling = useCallback(() => {
async function inputListen() {
// var numInputs = 0;
deviceSourceManager?.onDeviceConnectedObservable.add((device) => {
// numInputs++;
if (device.deviceType === DeviceType.Touch) {
const touch: DeviceSource<DeviceType.Touch> =
deviceSourceManager.getDeviceSource(
device.deviceType,
device.deviceSlot
)!;
touch.onInputChangedObservable.add((touchEvent) => {
if (
model &&
xrSession &&
placementIndicator?.isEnabled() &&
!modelPlaced
) {
placeModel();
//placementIndicator?.setEnabled(false);
console.log("model placing input");
//numInputs--;
}
if (
model &&
modelPlaced &&
xrSession &&
placementIndicator?.isEnabled() &&
touchEvent.previousState !== null &&
touchEvent.currentState !== null
) {
console.log("update input");
// Calculate the differential between two states.
const diff = touchEvent.previousState - touchEvent.currentState;
// Single input, do translation.
//if (numInputs === 1) {
if (touchEvent.inputIndex === PointerInput.Horizontal) {
model.position.x -= diff / 1000;
} else {
model.position.z += diff / 750;
}
// }
// Multi-input do rotation.
// if (
// numInputs === 2 &&
// touchEvent.inputIndex === PointerInput.Horizontal &&
// touchEvent.deviceSlot === 0
// ) {
// console.log("multi touch");
// model.rotate(Vector3.Up(), diff / 200);
// }
}
});
} else if (device.deviceType === DeviceType.Mouse) {
const mouse: DeviceSource<DeviceType.Mouse> =
deviceSourceManager.getDeviceSource(
device.deviceType,
device.deviceSlot
)!;
mouse.onInputChangedObservable.add((mouseEvent) => {
if (mouse.getInput(PointerInput.LeftClick)) {
return;
}
});
}
});
}
inputListen();
}, [xrSession, placementIndicator, modelPlaced]);
const reset2D = () => {
if (model && scene && camera) {
model.setEnabled(true);
model.position = camera.position.add(
camera.getForwardRay().direction.scale(scale * 1)
);
placementIndicator?.setEnabled(false);
setModelPlaced(false);
//model.scalingDeterminant = 0;
//camera.setTarget(model)
}
};
const placeModel = () => {
console.log("placeModel");
if (
xrSession &&
placementIndicator?.isEnabled() &&
scene &&
model &&
!modelPlaced
) {
setModelPlaced(true);
model.rotationQuaternion = Quaternion.Identity();
model.position = placementIndicator.position.clone();
placementIndicator?.setEnabled(false);
model.setEnabled(true);
//model.scalingDeterminant = 0;
}
};
const trackingStateToString = (
trackingState: WebXRTrackingState | undefined
): string => {
return trackingState === undefined ? "" : WebXRTrackingState[trackingState];
};
const onToggleXr = useCallback(() => {
(async () => {
if (xrSession) {
reset2D();
await xrSession.exitXRAsync();
} else {
if (rootNode !== undefined && scene !== undefined) {
const xr = await scene.createDefaultXRExperienceAsync({
disableDefaultUI: true,
disableTeleportation: true,
});
const session = await xr.baseExperience.enterXRAsync(
"immersive-ar",
"unbounded",
xr.renderTarget
);
setModelPlaced(false);
model?.setEnabled(false);
setXrSession(session);
session.onXRSessionEnded.add(() => {
setXrSession(undefined);
setTrackingState(undefined);
});
setTrackingState(xr.baseExperience.camera.trackingState);
xr.baseExperience.camera.onTrackingStateChanged.add(
(newTrackingState) => {
setTrackingState(newTrackingState);
}
);
// Set up the hit test.
const xrHitTestModule =
xr.baseExperience.featuresManager.enableFeature(
WebXRFeatureName.HIT_TEST,
"latest",
{
offsetRay: {
origin: { x: 0, y: 0, z: 0 },
direction: { x: 0, y: 0, z: -1 },
},
}
) as WebXRHitTest;
// Do some plane shtuff.
const xrPlanes = xr.baseExperience.featuresManager.enableFeature(
WebXRFeatureName.PLANE_DETECTION,
"latest"
) as WebXRPlaneDetector;
console.log("Enabled plane detection.");
const planes: any[] = [];
xrPlanes.onPlaneAddedObservable.add((webXRPlane) => {
if (scene) {
console.log("Plane added.");
let plane: any = webXRPlane;
webXRPlane.polygonDefinition.push(
webXRPlane.polygonDefinition[0]
);
try {
plane.mesh = MeshBuilder.CreatePolygon(
"plane",
{ shape: plane.polygonDefinition },
scene,
earcut
);
let tubeMesh: Mesh = TubeBuilder.CreateTube(
"tube",
{
path: plane.polygonDefinition,
radius: 0.005,
sideOrientation: Mesh.FRONTSIDE,
updatable: true,
},
scene
);
tubeMesh.setParent(plane.mesh);
planes[plane.id] = plane.mesh;
plane.mesh.material = planeMat;
plane.mesh.rotationQuaternion = new Quaternion();
plane.transformationMatrix.decompose(
plane.mesh.scaling,
plane.mesh.rotationQuaternion,
plane.mesh.position
);
} catch (ex) {
console.error(ex);
}
}
});
xrPlanes.onPlaneUpdatedObservable.add((webXRPlane) => {
console.log("Plane updated.");
let plane: any = webXRPlane;
if (plane.mesh) {
plane.mesh.dispose(false, false);
}
const some = plane.polygonDefinition.some((p: any) => !p);
if (some) {
return;
}
plane.polygonDefinition.push(plane.polygonDefinition[0]);
try {
plane.mesh = MeshBuilder.CreatePolygon(
"plane",
{ shape: plane.polygonDefinition },
scene,
earcut
);
let tubeMesh: Mesh = TubeBuilder.CreateTube(
"tube",
{
path: plane.polygonDefinition,
radius: 0.005,
sideOrientation: Mesh.FRONTSIDE,
updatable: true,
},
scene
);
tubeMesh.setParent(plane.mesh);
planes[plane.id] = plane.mesh;
plane.mesh.material = planeMat;
plane.mesh.rotationQuaternion = new Quaternion();
plane.transformationMatrix.decompose(
plane.mesh.scaling,
plane.mesh.rotationQuaternion,
plane.mesh.position
);
plane.mesh.receiveShadows = true;
} catch (ex) {
console.error(ex);
}
});
xrPlanes.onPlaneRemovedObservable.add((webXRPlane) => {
console.log("Plane removed.");
let plane: any = webXRPlane;
if (plane && planes[plane.id]) {
planes[plane.id].dispose();
}
});
xrHitTestModule.onHitTestResultObservable.add((results) => {
if (results.length) {
if (!modelPlaced) {
placementIndicator?.setEnabled(true);
} else {
placementIndicator?.setEnabled(false);
}
if (placementIndicator) {
placementIndicator.position = results[0].position;
}
}
});
}
}
})();
}, [
rootNode,
scene,
xrSession
]);
const onInitialized = useCallback(
async (engineViewCallbacks: EngineViewCallbacks) => {
setEngineViewCallbacks(engineViewCallbacks);
},
[engine]
);
const onSnapshot = useCallback(async () => {
if (engineViewCallbacks) {
setSnapshotData(
"data:image/jpeg;base64," + (await engineViewCallbacks.takeSnapshot())
);
}
}, [engineViewCallbacks]);
return (
<>
<View style={props.style}>
<Button
title="Toggle EngineView"
onPress={() => {
setToggleView(!toggleView);
}}
/>
<Button
title={xrSession ? "Stop XR" : "Start XR"}
onPress={() => {
//desabilitar o model assim que inicia a xrsession, no usecallback não funciona
if (!xrSession) {
model?.setEnabled(false);
}
onToggleXr();
}}
/>
{!toggleView && (
<View style={{ flex: 1 }}>
{enableSnapshots && (
<View style={{ flex: 1 }}>
<Button title={"Take Snapshot"} onPress={onSnapshot} />
<Image style={{ flex: 1 }} source={{ uri: snapshotData }} />
</View>
)}
<EngineView
style={props.style}
camera={camera}
onInitialized={onInitialized}
/>
<Slider
style={{
position: "absolute",
minHeight: 50,
margin: 10,
left: 0,
right: 0,
bottom: 0,
}}
minimumValue={0.2}
maximumValue={2}
step={0.01}
value={defaultScale}
onValueChange={setScale}
/>
<Text
style={{
fontSize: 12,
color: "yellow",
position: "absolute",
margin: 10,
}}
>
{trackingStateToString(trackingState)}
</Text>
</View>
)}
{toggleView && (
<View
style={{ flex: 1, justifyContent: "center", alignItems: "center" }}
>
<Text style={{ fontSize: 24 }}>EngineView has been removed.</Text>
<Text style={{ fontSize: 12 }}>
Render loop stopped, but engine is still alive.
</Text>
</View>
)}
</View>
</>
);
};
const BabylonXrTest = ({ route, navigation }) => {
const [toggleScreen, setToggleScreen] = useState(false);
//const { topic } = route.params;
return (
<>
<SafeAreaView style={{ flex: 1, backgroundColor: "white" }}>
{!toggleScreen && <EngineScreen style={{ flex: 1 }} />}
{toggleScreen && (
<View
style={{ flex: 1, justifyContent: "center", alignItems: "center" }}
>
<Text style={{ fontSize: 24 }}>EngineScreen has been removed.</Text>
<Text style={{ fontSize: 12 }}>
Engine has been disposed, and will be recreated.
</Text>
</View>
)}
<Button
title="Toggle EngineScreen"
onPress={() => {
setToggleScreen(!toggleScreen);
}}
/>
</SafeAreaView>
</>
);
};
export default BabylonXrTest;
const styles = StyleSheet.create({
engineView: {
backgroundColor: "transparent",
},
});