Fixing a 3D model in an augmented reality environment

Questions
1

Hello,
I’m creating an augmented reality app using BABYLONJS which consists of fixing the position of a 3D model by getting the ‘z’ and ‘x’ values ​​from a HitTest on a vertical surface and the ‘y’ value on a horizontal surface in the same way.
I’ve already managed to get this working but when I test it on a mobile device and move away from the detected surfaces, the model moves slowly with me, however, I want the model to remain fixed in the defined position.

button.onPointerDownObservable.add(async function () {
    if (marker.isVisible) {
      let position = marker.position.clone();
      let rotation = marker.rotationQuaternion.clone();

      if (!isVertical) {
        storedPosition.y = position.y;
        hFound = true;
      } else {
        console.log("Vertical: " + marker.rotationQuaternion);
        storedRotation = rotation;
        storedPosition.x = position.x;
        storedPosition.z = position.z;
        vFound = true;
      }
      if (hFound && vFound) {
        model.position = storedPosition.clone();

        const forward = normal.clone();
        const up = BABYLON.Vector3.Up();
        const right = BABYLON.Vector3.Cross(up, forward).normalize();

        const adjustedUp = BABYLON.Vector3.Cross(forward, right).normalize();

        const rotationMatrix = new BABYLON.Matrix();
        BABYLON.Matrix.FromXYZAxesToRef(
          right,
          adjustedUp,
          forward,
          rotationMatrix
        );
        const finalRotation =
          BABYLON.Quaternion.FromRotationMatrix(rotationMatrix);
        model.rotationQuaternion = finalRotation;

        const rotation180Y = BABYLON.Quaternion.RotationAxis(
          new BABYLON.Vector3(0, 1, 0),
          BABYLON.Tools.ToRadians(180)
        );
        model.rotationQuaternion = rotation180Y.multiply(
          model.rotationQuaternion
        );
        
        xrAnchors
          .addAnchorAtPositionAndRotationAsync(
            model.position,
            model.rotationQuaternion
          )
          .then((anchor) => {
            anchor.attachedNode = model;
            console.log(anchor);
          });
      }
    }
  });

I would like to know what would be the best way to fix the model to the floor and wall and keep it fixed throughout the augmented reality experience in a precise way.

Thanks for the help!

2

If you are using anchors (which you are) the underlying system is in charge of maintaining the anchor’s position over time. If there is a slight drift, it is sadly the OS miscalculating the new position of the object. This can happen in an environment that is a bit “confusing” to the camera - lack of textures for example (i.e. - only white walls), or when moving fast from one room to the other. How heavy is the drift?

3

I understand that there may be some drift when moving quickly between rooms. Also, the issue you mention about camera focus on white walls is significant as the tests have been done mostly on white walls. However, the issue is with the object’s perspective as what I would normally expect is that as I move away from the 3d model the object would appear smaller rather than moving towards me while maintaining the same size. Imagine the 3d model is a painting on a wall, I would like the further away I am from the wall the smaller the painting would be.

That being said, I would like to know if I have implemented the anchoring system correctly in order to achieve my goal.

xrAnchors
          .addAnchorAtPositionAndRotationAsync(
            model.position,
            model.rotationQuaternion
          )
          .then((anchor) => {
            anchor.attachedNode = model;
            console.log(anchor);
          });

Is there anything else I should keep in mind to get the anchoring system set up well?

4

No, this is the way to go. You are creating an anchor in a specific position, and then attaching a mesh to it. The anchor will be the mesh’s parent and will make sure the mesh will always stay in the right place.

6

I couldn’t get the object to not move using anchors, maybe I’m missing something. I added the “onAnchorAddedObservable” method but when creating the anchor and detecting the event the model rotates 180 degrees on the Y axis.

if (anchors) {
    anchors.onAnchorAddedObservable.add((anchor) => {
      model.setEnabled(true);
      anchor.attachedNode = model.clone("clone");
      model.setEnabled(false);
    });

    anchors.onAnchorRemovedObservable.add((anchor) => {
      if (anchor) {
        anchor.attachedNode.isVisible = false;
        anchor.attachedNode.dispose();
      }
    });
  }

button.onPointerDownObservable.add(async function () {
    if (marker.isVisible) {
      let position = marker.position.clone();
      let rotation = marker.rotationQuaternion.clone();

      if (!isVertical) {
        storedPosition.y = position.y;
        hFound = true;
      } else {
        console.log("Vertical: " + marker.rotationQuaternion);
        storedRotation = rotation;
        storedPosition.x = position.x;
        storedPosition.z = position.z;

        forward = normal.clone();
        up = BABYLON.Vector3.Up();
        right = BABYLON.Vector3.Cross(up, forward).normalize();

        vFound = true;
      }
      if (hFound && vFound) {
        model.position = storedPosition.clone();
        const adjustedUp = BABYLON.Vector3.Cross(forward, right).normalize();

        const rotationMatrix = new BABYLON.Matrix();
        BABYLON.Matrix.FromXYZAxesToRef(
          right,
          adjustedUp,
          forward,
          rotationMatrix
        );
        const finalRotation =
          BABYLON.Quaternion.FromRotationMatrix(rotationMatrix);
        model.rotationQuaternion = finalRotation;

        const rotation180Y = BABYLON.Quaternion.RotationAxis(
          new BABYLON.Vector3(0, 1, 0),
          BABYLON.Tools.ToRadians(180)
        );
        model.rotationQuaternion = rotation180Y.multiply(
          model.rotationQuaternion
        );

        anchors.addAnchorAtPositionAndRotationAsync(
          model.position,
          model.rotationQuaternion
        );
        console.log(model.position, model.rotationQuaternion);
      }
    }
  });