Can WebXR captureImage in AR mode for computer vision

Questions
1

Is it possible to perform a grabFrame() call when in an AR session? I am trying to grab from the XRWebGlLayer that is used during the WebXR session.

function init {
navigator.mediaDevices.enumerateDevices()
.then((devices) => {
if (window.stream) {
window.stream.getTracks().forEach(track => {
track.stop();
});
}
devices.forEach((device) => {
console.log(device.kind + ": " + device.label + " id = " + device.deviceId);

                if (device.kind === 'videoinput') {
                    constraints = {
                        video: { facingMode: { exact: "environment" } }
                    }
                }
            });
            navigator.mediaDevices.getUserMedia(constraints)
                .then((stream) => {
                    window.stream = stream; // make stream available to console
                    videoElement.srcObject = stream;
                    track = window.stream.getVideoTracks()[0];
                    var imageCapture = new ImageCapture(track);
                    interval = setInterval(() => {
                        imageCapture.grabFrame()
                            .then((imgData) => {
                                canvas.width = imgData.width;
                                canvas.height = imgData.height;
                                canvas.getContext('2d').drawImage(imgData, 0, 0);
                            })
                            .catch(err => console.error('grabFrame() failed: ', err));
                    }, 1000);
                })
                .catch(handleError);
        })
        .catch((e) => {
            console.log(e.name + ": " + e.message);
        });

}

2

@RaananW may probably help you for this one.

3

Hey, sorry - was away.

What information do you need? The XR WebGLLayer is a private member of the xr session manager. It is also a member of the renderTarget public member of the xr default experience helper.

If you need to camera details (or - what the device “sees” - this ifnormation is, as far as I know, not available.

4

Thank you for the response. We wanted to use the camera image during the XR session to use in a js computer vision library that works with the 2d camera image. I found a few docs requesting an image grab feature in WebXR. It looks like something they are working for future releases.

The request

Here is someone’s notes on what they accomplished.

1 Like