Babylon.js 5 changed simulated mouse movement behavior

Bugs
1

Hey,

I’m adding the topic as a bug because it’s something that works on 4.2.1.
I’m using testcafe test runner as a tool to automate some user actions and create e2e tests for 3d project. So far it worked pretty well but when I checked mouse drag movement simulation on 5.0.0 it is no longer working, you can see it in video below:

Maybe something changed that actually should be updated/changed on testcafe side I’m not aware of?
If you’d like to reproduce the issue I used this playground as nice place to debug:

And test code would be (long):

// @ts-nocheck
import { ClientFunction } from "testcafe";

//long func to get sphere location on screen
let getSphere = ClientFunction(() => {
  let tokenMesh = BABYLON.Inspector._Scene.getNodeByName("sphere1");
  let vector3 = BABYLON.Vector3;
  scene.preventDefaultOnPointerDown;
  const coordinates = tokenMesh._boundingInfo.boundingBox.vectors.map(
    (vector) =>
      vector3.Project(
        vector,
        tokenMesh._worldMatrix,
        BABYLON.Inspector._Scene._transformMatrix,
        BABYLON.Inspector._Scene.activeCamera.viewport.toGlobal(
          BABYLON.Inspector._Scene._engine.getRenderWidth(),
          BABYLON.Inspector._Scene._engine.getRenderHeight()
        )
      )
  );

  const minMax = {
    min: coordinates[0],
    max: coordinates[0],
  };

  for (let i = 1; i < coordinates.length; i++) {
    minMax.min = vector3.Minimize(minMax.min, coordinates[i]);
    minMax.max = vector3.Maximize(minMax.max, coordinates[i]);
  }

  let result = {
    x: Math.round((minMax.max.x + minMax.min.x) / 2),
    y: Math.round((minMax.max.y + minMax.min.y) / 2),
  };

  return result;
});

//waiter for scene to load
let isSceneReady = ClientFunction(() => {
  let isRdy = false;
  let pendingData = BABYLON.Inspector._Scene._pendingData.length;
  let pendingAnimation = BABYLON.Inspector._Scene.animatables.length;

  if (pendingData + pendingAnimation == 0) {
    isRdy = true;
  }
  console.log("scene rdy: " + isRdy);
  return isRdy;
});

fixture`DM token move`
  .page("https://playground.babylonjs.com/#8UQCQI#19")
  .beforeEach(async (t) => {
    //waiter till page and scene loads
    await t
      .wait(5000)
      .expect(await isSceneReady())
      .ok({ timeout: 70000 });
  });

test("movement in 5.0", async (t) => {
  await t.debug();

  while (true) {
    let sphere = await getSphere();

    await t
    .click("#renderCanvas", {
      offsetX: sphere.x,
      offsetY: sphere.y,
    })
    .drag("#renderCanvas", 100, 0, {
      offsetX: sphere.x,
      offsetY: sphere.y,
      speed: 0.2,
    })
    .debug()
    .wait(2000);
  }
});

If you’d like to run its enough to:
testcafe chrome movement.ts --skip-js-errors

I’m still investigating this, will add more info if I find something helpful.

2

cc @PolygonalSun

3

Hey @Unkas, thanks for all the info on the find! So between 4.2 and 5, we modified the input system to work with an intermediate system to enable Babylon Native to work with various JS input classes. It also does some routing and handling of event data so I’m wondering if there’s a potential disconnect with how testcafe is dispatching events or if maybe the DeviceInputSystem (the new system) is misinterpreting the input data. When you run the test in the video you provided, what does the console log look like?

4

@PolygonalSun

I made slightly updated recording with attached playground so now we can see that pointer down is being somewhat understood (ball goes transparent) but movement does not work as intended (though when pointer goes out of the ball it moves the camera in 5.0):

Here are logs from this recording, I expanded pointerdown and pointermove event in each of them:

5.0

Babylon.js v5.0.0-beta.7 - WebGL2
babylon.js:17 getGamepad will now require Secure Context. Please update your application accordingly. For more information see https://github.com/w3c/gamepad/pull/120
e._checkForConnectedDevices @ babylon.js:17
e._configureEvents @ babylon.js:17
e @ babylon.js:17
e @ babylon.js:17
e._Create @ babylon.js:17
_ @ babylon.js:17
e.attachControl @ babylon.js:17
t.attachControl @ babylon.js:17
t @ babylon.js:17
createScene @ VM46:5
window.initFunction @ VM46:97
hammerhead.js:15 scene rdy: true
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 2, previousState: 0, …}
hammerhead.js:15 POINTER DOWN
hammerhead.js:15 POINTER PICK
hammerhead.js:15 POINTER TAP
hammerhead.js:15 POINTER UP
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 2, previousState: 0, …}currentState: 1deviceSlot: 0deviceType: 2inputIndex: 2isTrusted: falsepreviousState: 0altKey: falsealtitudeAngle: 1.5707963267948966azimuthAngle: 0bubbles: truebutton: 0buttons: 1cancelBubble: falsecancelable: trueclientX: 931clientY: 353composed: truectrlKey: falsecurrentTarget: nulldefaultPrevented: truedetail: 1eventPhase: 0fromElement: nullheight: 1isPrimary: truelayerX: 209layerY: 298metaKey: falsemovementX: 0movementY: 0offsetX: 209offsetY: 298pageX: 931pageY: 353path: (10) [canvas#renderCanvas, div, div#canvasZone.pg-split-part.canvasZone, div#pg-split, div#pg-root, div#host-element, body, html, document, Window]pointerId: 1pointerType: "mouse"pressure: 0relatedTarget: bodyreturnValue: falsescreenX: 0screenY: 0shiftKey: falsesourceCapabilities: nullsrcElement: canvas#renderCanvastangentialPressure: 0target: canvas#renderCanvastiltX: 0tiltY: 0timeStamp: 26129.19999998808toElement: nulltwist: 0type: "pointerdown"view: Window {window: Window, self: Window, document: document, name: '', location: Location, …}which: 1width: 1x: 931y: 353[[Prototype]]: PointerEvent
hammerhead.js:15 POINTER DOWN
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}currentState: 0deviceSlot: 0deviceType: 2inputIndex: 2isTrusted: falsepreviousState: 1altKey: falsealtitudeAngle: 1.5707963267948966azimuthAngle: 0bubbles: truebutton: 0buttons: 1cancelBubble: falsecancelable: trueclientX: 932clientY: 352composed: truectrlKey: falsecurrentTarget: nulldefaultPrevented: truedetail: 0eventPhase: 0fromElement: nullheight: 1isPrimary: truelayerX: 210layerY: 297metaKey: falsemovementX: 0movementY: 0offsetX: 210offsetY: 297pageX: 932pageY: 352path: (10) [canvas#renderCanvas, div, div#canvasZone.pg-split-part.canvasZone, div#pg-split, div#pg-root, div#host-element, body, html, document, Window]pointerId: 1pointerType: "mouse"pressure: 0relatedTarget: bodyreturnValue: falsescreenX: 932screenY: 352shiftKey: falsesourceCapabilities: nullsrcElement: canvas#renderCanvastangentialPressure: 0target: canvas#renderCanvastiltX: 0tiltY: 0timeStamp: 26457.399999976158toElement: nulltwist: 0type: "pointermove"view: Window {window: Window, self: Window, document: document, name: '', location: Location, …}which: 1width: 1x: 932y: 352[[Prototype]]: PointerEvent
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 POINTER PICK
hammerhead.js:15 POINTER TAP
hammerhead.js:15 POINTER UP
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 2, previousState: 0, …}
hammerhead.js:15 POINTER DOWN
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 POINTER PICK
hammerhead.js:15 POINTER TAP
hammerhead.js:15 POINTER UP
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 2, previousState: 0, …}
hammerhead.js:15 POINTER DOWN
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 POINTER DOUBLE-TAP
hammerhead.js:15 POINTER UP
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 2, previousState: 0, …}
hammerhead.js:15 POINTER DOWN
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 POINTER PICK
hammerhead.js:15 POINTER TAP
hammerhead.js:15 POINTER UP
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 2, previousState: 0, …}
hammerhead.js:15 POINTER DOWN
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 POINTER DOUBLE-TAP
hammerhead.js:15 POINTER UP
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 2, previousState: 0, …}
hammerhead.js:15 POINTER DOWN
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 POINTER PICK
hammerhead.js:15 POINTER TAP
hammerhead.js:15 POINTER UP
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 2, previousState: 0, …}
hammerhead.js:15 POINTER DOWN
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 POINTER DOUBLE-TAP
hammerhead.js:15 POINTER UP
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 2, previousState: 0, …}
hammerhead.js:15 POINTER DOWN
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 POINTER PICK
hammerhead.js:15 POINTER TAP
hammerhead.js:15 POINTER UP
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 2, previousState: 0, …}
hammerhead.js:15 POINTER DOWN
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 POINTER DOUBLE-TAP
hammerhead.js:15 POINTER UP
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 2, previousState: 0, …}
hammerhead.js:15 POINTER DOWN
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 POINTER PICK
hammerhead.js:15 POINTER TAP
hammerhead.js:15 POINTER UP
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 12, pointerId: 1, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, deviceType: 2, deviceSlot: 0, inputIndex: 2, previousState: 0, …}
hammerhead.js:15 POINTER DOWN
hammerhead.js:15 POINTER PICK
hammerhead.js:15 POINTER TAP
hammerhead.js:15 POINTER UP

4.2

Babylon.js v4.2.1 - WebGL2
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER DOWN
hammerhead.js:15 POINTER PICK
hammerhead.js:15 POINTER TAP
hammerhead.js:15 POINTER UP
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}isTrusted: falsealtKey: falsealtitudeAngle: 1.5707963267948966azimuthAngle: 0bubbles: truebutton: 0buttons: 1cancelBubble: falsecancelable: trueclientX: 931clientY: 353composed: truectrlKey: falsecurrentTarget: nulldefaultPrevented: truedetail: 1eventPhase: 0fromElement: nullheight: 1isPrimary: truelayerX: 209layerY: 298metaKey: falsemovementX: 0movementY: 0offsetX: 209offsetY: 298pageX: 931pageY: 353path: (10) [canvas#renderCanvas, div, div#canvasZone.pg-split-part.canvasZone, div#pg-split, div#pg-root, div#host-element, body, html, document, Window]pointerId: 1pointerType: "mouse"pressure: 0relatedTarget: bodyreturnValue: falsescreenX: 0screenY: 0shiftKey: falsesourceCapabilities: nullsrcElement: canvas#renderCanvastangentialPressure: 0target: canvas#renderCanvastiltX: 0tiltY: 0timeStamp: 16160.5toElement: nulltwist: 0type: "pointerdown"view: Window {window: Window, self: Window, document: document, name: '', location: Location, …}which: 1width: 1x: 931y: 353[[Prototype]]: PointerEvent
hammerhead.js:15 POINTER DOWN
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}isTrusted: falsealtKey: falsealtitudeAngle: 1.5707963267948966azimuthAngle: 0bubbles: truebutton: 0buttons: 1cancelBubble: falsecancelable: trueclientX: 932clientY: 352composed: truectrlKey: falsecurrentTarget: nulldefaultPrevented: falsedetail: 0eventPhase: 0fromElement: nullheight: 1isPrimary: truelayerX: 210layerY: 297metaKey: falsemovementX: 0movementY: 0offsetX: 210offsetY: 297pageX: 932pageY: 352path: (10) [canvas#renderCanvas, div, div#canvasZone.pg-split-part.canvasZone, div#pg-split, div#pg-root, div#host-element, body, html, document, Window]pointerId: 1pointerType: "mouse"pressure: 0relatedTarget: bodyreturnValue: truescreenX: 932screenY: 352shiftKey: falsesourceCapabilities: nullsrcElement: canvas#renderCanvastangentialPressure: 0target: canvas#renderCanvastiltX: 0tiltY: 0timeStamp: 16492.100000023842toElement: nulltwist: 0type: "pointermove"view: Window {window: Window, self: Window, document: document, name: '', location: Location, …}which: 1width: 1x: 932y: 352[[Prototype]]: PointerEvent
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 PointerEvent {isTrusted: false, pointerId: 1, width: 1, height: 1, pressure: 0, …}
hammerhead.js:15 POINTER MOVE
hammerhead.js:15 POINTER UP

Main difference i see for now are these parameters in 5.0 PointerEvent, they are absent in 4.2:

currentState: 0
deviceSlot: 0
deviceType: 2
inputIndex: 2
previousState: 1

In 5.0 example “true” mouse movement has this params like this:

deviceSlot: 0
deviceType: 2
inputIndex: 12

Aha, it’s all recorded on Chrome, but I tried with FF and it looks the same.

5

After a bit more investigation I think I need to learn more about test runners simulating movement. Anyway, testcafe during test has this layer that needs to be unlocked to interact with page with “true” mouse. I just checked Playwright and movement works there. Event in logs look way closer to true mouse, but then that might be difference in approach between two solutions.

Playwright move event:

Babylon.js v5.0.0-beta.7 - WebGL2 - Parallel shader compilation
babylon.js:16 The AudioContext was not allowed to start. It must be resumed (or created) after a user gesture on the page. https://goo.gl/7K7WLu
e._initializeAudioContext @ babylon.js:16
get @ babylon.js:16
e._afterRender @ babylon.js:16
t.render @ babylon.js:16
(anonymous) @ babylon.playground.js:16
t._renderFrame @ babylon.js:16
t._renderLoop @ babylon.js:16
VM34:55 PointerEvent
VM34:56 POINTER MOVE
VM34:55 PointerEventdeviceSlot: 0deviceType: 2inputIndex: 12isTrusted: truealtKey: falsealtitudeAngle: 1.5707963267948966azimuthAngle: 0bubbles: truebutton: -1buttons: 0cancelBubble: falsecancelable: trueclientX: 887.85546875clientY: 579.65234375composed: truectrlKey: falsecurrentTarget: nulldefaultPrevented: falsedetail: 0eventPhase: 0fromElement: nullheight: 1isPrimary: truelayerX: 887layerY: 524metaKey: falsemovementX: 2movementY: 0offsetX: 245.85546875offsetY: 524.65234375pageX: 887.85546875pageY: 579.65234375path: (10) [canvas#renderCanvas, div, div#canvasZone.pg-split-part.canvasZone, div#pg-split, div#pg-root, div#host-element, body, html, document, Window]pointerId: 1pointerType: "mouse"pressure: 0relatedTarget: nullreturnValue: truescreenX: 909.85546875screenY: 705.65234375shiftKey: falsesourceCapabilities: nullsrcElement: canvas#renderCanvastangentialPressure: 0target: canvas#renderCanvastiltX: 0tiltY: 0timeStamp: 3561.900000035763toElement: nulltwist: 0type: "pointermove"view: Window {window: Window, self: Window, document: document, name: '', location: Location, …}which: 0width: 1x: 887.85546875y: 579.65234375[[Prototype]]: PointerEvent
VM34:56 POINTER MOVE
VM34:55 PointerEventdeviceSlot: 0deviceType: 2inputIndex: 12isTrusted: truealtKey: falsealtitudeAngle: 1.5707963267948966azimuthAngle: 0bubbles: truebutton: -1buttons: 0cancelBubble: falsecancelable: trueclientX: 896.84375clientY: 579.37109375composed: truectrlKey: falsecurrentTarget: nulldefaultPrevented: falsedetail: 0eventPhase: 0fromElement: nullheight: 1isPrimary: truelayerX: 896layerY: 524metaKey: falsemovementX: 9movementY: 0offsetX: 254.84375offsetY: 524.37109375pageX: 896.84375pageY: 579.37109375path: (10) [canvas#renderCanvas, div, div#canvasZone.pg-split-part.canvasZone, div#pg-split, div#pg-root, div#host-element, body, html, document, Window]pointerId: 1pointerType: "mouse"pressure: 0relatedTarget: nullreturnValue: truescreenX: 918.84375screenY: 705.37109375shiftKey: falsesourceCapabilities: nullsrcElement: canvas#renderCanvastangentialPressure: 0target: canvas#renderCanvastiltX: 0tiltY: 0timeStamp: 3574.100000023842toElement: nulltwist: 0type: "pointermove"view: Window {window: Window, self: Window, document: document, name: '', location: Location, …}which: 0width: 1x: 896.84375y: 579.37109375[[Prototype]]: PointerEvent
VM34:56 POINTER MOVE
VM34:55 PointerEvent
VM34:56 POINTER MOVE
6

Update:
It seems that core issue here is that now mouse movement loses drag for some reason. Maybe I would need to somehow force “move with click” ?
There is this POINTER UP event when drag starts - as if pointer down is finished as soon as another pointer event happens (move in this case).

Anyway here how it looks when I’m monitoring drag state, and then forcing drag on pointer down.
Video:

8

@PolygonalSun will be able to look into it shortly

9

So I’ve set up testcafe on a machine and I’m going to try and run it against a local instance of Babylon.js to see if I can understand what’s going on with the events

10

When tried to run the script given as specified above, I got the following error once the test executed:

 Running tests in:
 - Chrome 98.0.4758.102 / Windows 10

 DM token move
node:internal/console/constructor:291
          throw e;
          ^
11

Hey ! Thanks for trying it out - even if there was an error. Only reason why it may fail that I know of is my clumsy waiting for page to load (by just wait x seconds) its defined here and can be changed from 5000 to lets say 10000)

fixture`DM token move`
  .page("https://playground.babylonjs.com/#8UQCQI#19")
  .beforeEach(async (t) => {
    //waiter till page and scene loads
    await t
      .wait(10000)
      .expect(await isSceneReady())
      .ok({ timeout: 70000 });
  });

In the meantime I think I found some solution, instead of using .drag method I can use dispatchEvent:

.dispatchEvent("#renderCanvas", "pointerdown", {
        clientX: sphere.x + 20,
        clientY: sphere.y,
      })

This however works only on 5.00 and not in 4.2 version. It’s more workaround than finding the rootcause but if the code still doesn’t work for you - I have now working solution and I’m not blocked anymore :slight_smile:

1 Like
12

From some reason, when I hit resume, the test is performed and then things crash with the previously mentioned error and a “RangeError: Maximum call stack size exceeded” message. While I wasn’t able to get it to work for me, I did find something that I was curious about. In the test, you call click before you call drag. Is that testcafe performing a click event? The reason I’m asking is because a click event would explain a pointerup occurrence. When a click event happens, a pointerdown event is dispatch and then a pointerup is dispatched.

1 Like
13

Yep there was a click before drag, but pointer up happens while dragging - Just in case I updated the code.

Here is updated code that is working for me everytime, removed clumsy wait and click before drag to avoid any confusion. Removed also infinite while as maybe this was causing issues, so now after drag the test will stop, when resumed it will finish and will need to be ran again:

// @ts-nocheck
import { Selector, ClientFunction } from "testcafe";

//long func to get sphere location on screen
let getSphere = ClientFunction(() => {
  let tokenMesh = BABYLON.Inspector._Scene.getNodeByName("sphere1");
  let vector3 = BABYLON.Vector3;
  scene.preventDefaultOnPointerDown;
  const coordinates = tokenMesh._boundingInfo.boundingBox.vectors.map(
    (vector) =>
      vector3.Project(
        vector,
        tokenMesh._worldMatrix,
        BABYLON.Inspector._Scene._transformMatrix,
        BABYLON.Inspector._Scene.activeCamera.viewport.toGlobal(
          BABYLON.Inspector._Scene._engine.getRenderWidth(),
          BABYLON.Inspector._Scene._engine.getRenderHeight()
        )
      )
  );

  const minMax = {
    min: coordinates[0],
    max: coordinates[0],
  };

  for (let i = 1; i < coordinates.length; i++) {
    minMax.min = vector3.Minimize(minMax.min, coordinates[i]);
    minMax.max = vector3.Maximize(minMax.max, coordinates[i]);
  }

  let result = {
    x: Math.round((minMax.max.x + minMax.min.x) / 2),
    y: Math.round((minMax.max.y + minMax.min.y) / 2),
  };

  return result;
});

//waiter for scene to load
let isSceneReady = ClientFunction(() => {
  let isRdy = false;
  let pendingData = BABYLON.Inspector._Scene._pendingData.length;
  let pendingAnimation = BABYLON.Inspector._Scene.animatables.length;

  if (pendingData + pendingAnimation == 0) {
    isRdy = true;
  }
  console.log("scene rdy: " + isRdy);
  return isRdy;
});

fixture`movement`
  .page("https://playground.babylonjs.com/#8UQCQI#22")
  .beforeEach(async (t) => {
    //waiter till page and scene loads
    const header = Selector('div').withAttribute('id', 'pg-header');
    await t
      .expect(header.exists)
      .ok({ timeout: 10000 })
      .expect(await isSceneReady())
      .ok({ timeout: 70000 });
  });

test("movement in 5.0", async (t) => {
    let sphere = await getSphere();

    await t
    .drag("#renderCanvas", 100, 0, {
      offsetX: sphere.x,
      offsetY: sphere.y,
      speed: 0.2
    })
    .debug()
    .wait(2000);
});

I run is like this (considering code is in test-file.ts) :

testcafe 'chrome --start-fullscreen' test-file.ts --skip-js-errors

If however it still drops errors on you then don’t bother - I don’t want to consume too much of your time. If that will be the case I will use workaround solution and mark it as solved :slight_smile: