Using observables for lip syncing

I was wondering does anyone know of a way to make a 3d models mouth move based on what vowels are displayed in a text box because I’m trying to make a 3d models mouth move based on the visemes I got from Amazon polly but I’m having trouble figuring out a way to make it so that my 3d models mouth will move based on what is obtained. If you need a visual example I’ve attached one to this topic. And for another example when the value = i the 3d models mouth will move and make a shape when a person says a word with an i vowel in it (or in this case lip syncing).
Screenshot 2022-01-01 192211

Do you have all animations for this in your model?

1 Like

Yes I do

If you would be able to create a repro in PG it would be much easier for everybody to help you.

Okay I’ve posted my babylonjs code and the code I use to obtain the visemes
function speakText() {

        // Create the JSON parameters for getSynthesizeSpeechUrl
        var speechParams = {
            OutputFormat: "json",
            SampleRate: "16000",
            Text: "",
            TextType: "text",
            VoiceId: "Joanna",
            SpeechMarkTypes: ["viseme", "word"]
        };
        speechParams.Text = document.getElementById("textEntry").value.toLowerCase(); 
        console.log(speechParams.Text);
        
        // Create the Polly service object and presigner object
        var polly = new AWS.Polly({apiVersion: '2016-06-10'});
        var signer = new AWS.Polly.Presigner(speechParams, polly);

        // Create presigned URL of synthesized speech file
        signer.getSynthesizeSpeechUrl(speechParams, function(error, url) {
        if (error) {
            document.getElementById('result').innerHTML = error;
        } else {
            
            console.log(url);
            console.log(signer);
            
            //var selectedText = "";
            var bracket = "}";
            var jsonIndex = -1;
            var itemHTML = "";
            var hash;
            //var myJson = [];
            
            fetch(url, headers => 'Content-Type:', 'application/json').then(res => res.text()).then((out) => {
                var myJson = "["+out;
                console.log(myJson);
                //For every index of a end curly bracket } add a comma to the right of it except the last curly bracket.
                for(let j = 0; j < myJson.length; j++){
                    var str = myJson.replace(/(\n)/g,",\n");
                    console.log(str);
                }
                //Gets the last index of a comma and removes it
                var newJson = str.lastIndexOf(",");
                str = str.substring(0, newJson);
                finalJson = str+"]";
               
                var jsonParsed = JSON.parse(finalJson);
                console.log(jsonParsed);
                console.log(jsonParsed[0]['value']);
                console.log(jsonParsed.length);
                for(var i = 0; i < jsonParsed.length; i++){
                    viseme = jsonParsed[i]['value'];
                    if(viseme.includes('e')){console.log("got it");}
                    if(viseme.includes('a')){console.log("Gimme an a");}
                    if(viseme.includes('i')){console.log("Here's an e");}
                    if(viseme.includes('u')){console.log("Found a u");}
                    if(viseme.includes('o')){console.log("I see an o");}
                    time = jsonParsed[i]['time'];
                    startTime = jsonParsed[i]['start'];
                    endTime = jsonParsed[i]['end'];
                    console.log(viseme);
                    console.log(time);
                    console.log(startTime);
                    console.log(endTime);
                    
                }
            }).catch(error => {throw error});
            /*document.getElementById('audioSource').src = url;
            document.getElementById('audioPlayback').load();
            document.getElementById('result').innerHTML = "Speech ready to play.";
            document.getElementById('playButton');*/
        }
      });
        }

var createScene = function () {
var scene = new BABYLON.Scene(engine);
//var layer = new BABYLON.Layer(’’, “Cyber_Background.jpg”, scene, true);
//var hdrTexture = BABYLON.CubeTexture.CreateFromPrefilteredData(“textures/environment.dds”, scene);
//var hdrTexture = BABYLON.CubeTexture.CreateFromPrefilteredData(“Cyber_Background.jpg”, scene);
//var currentSkybox = scene.createDefaultSkybox(hdrTexture, true);
//var camera = new BABYLON.FreeCamera(“camera1”, new BABYLON.Vector3(0, 5, -10), scene);
var camera = new BABYLON.ArcRotateCamera(“camera1”, Math.PI / -2, 1, 3, new BABYLON.Vector3(0, 3, 0), scene);
camera.attachControl(canvas, true);
//camera.position = new BABYLON.Vector3(0, 50, -40);
var light = new BABYLON.HemisphericLight(“light1”, new BABYLON.Vector3(0, 1, 0), scene);
light.intensity = 0.7;
var ground = BABYLON.Mesh.CreateGround(“ground1”, 6, 6, 2, scene);
ground.receiveShadows = true;
var gui;

    var scales = [];
var url;
var fileName;
    
    url = 'https://raw.githubusercontent.com/chris45242/BabylonModel/main/';
    fileName = "project.blend1.gltf";

    var Casi = BABYLON.SceneLoader.Append(url, fileName, scene, function (scene){
            // Create a default arc rotate camera and light.
            scene.createDefaultCameraOrLight(true, true, true);
            scene.activeCamera.alpha += Math.PI;
            
            scene.stopAllAnimations();

	scene.animationGroups[0].start(true); 
            scene.animationGroups[1].start(true);

            //Get's some of the meshes that are used to make Casi
            var casiBody = scene.getMeshByName("Casi's Body.001_primitive0");
            var casiInnerMouth = scene.getMeshByName("Casi's Body.001_primitive2");
            /*casiInnerMouth.setEnabled(false);
            var casiEyes = scene.getMeshByName("Casi's Body.001_primitive1");
            var casiTeeth = scene.getMeshByName("Casi's Teeth");*/
            var primitive = scene.getMeshByName("Primitives.001");
            
            //Gets Casi's Visor and uses visibility to make it see through glass-like material
            var casiVisor = scene.getMeshByName("Casi's Visor");
            casiVisor.setEnabled(true);
            casiVisor.visibility = 0.7;
            
            //Set up Morph Targets for Casi before the screen is done loading.
            let t = 0;
            var speak = document.getElementById("speaker").innerHTML;
            console.log(speak);
            lipSync = scene.onBeforeRenderObservable.add(function(){
                //casiBody.morphTargetManager.getTarget(1).influence = Math.abs(Math.sin(t));
                primitive.morphTargetManager.getTarget(2).influence = Math.abs(Math.sin(t));
                casiBody.morphTargetManager.getTarget(6).influence = Math.abs(Math.sin(t));
                //casiBody.morphTargetManager.getTarget(6).influence = 1;
                casiInnerMouth.morphTargetManager.getTarget(6).influence = Math.abs(Math.sin(t));
                //var getViseme = viseme;
                //console.log(getViseme);
                //casiInnerMouth.morphTargetManager.getTarget(6).influence = 1;
                //casiBody.morphTargetManager.getTarget(0).influence = Math.abs(Math.cos(t));
                t += 0.07;
        });

});
return scene;

};

var engine = new BABYLON.Engine(canvas, true, { preserveDrawingBuffer: true, stencil: true });

var scene = createScene();

engine.runRenderLoop(function () {
if (scene) {
scene.render();
}
});

Hi, @Chris1 !

I think, custom observables are helpful for your case.
I made a simple PG with it.

I hope it helps!

3 Likes

Hello there! There is an article about that here:

:smiley:

When I tried it the vowelInput.onValueChange = new BABYLON.Observable(); worked but everything else beneath it didn’t. Am I missing something?

Would be great if you could share a repro ?

Do you mean a playground or git repository?

A playground would be great ?

I’m still a little new to BabylonJS do you know how I can post a playground link because I still haven’t figured that out

yup you can go here https://playground.babylonjs.com/ create the repro of the issue, save and copy the new url after saving in here :slight_smile:

Here’s a playground of my problem

Your code should be in the callback :slight_smile: https://playground.babylonjs.com/#VU1G98#6

It works thank you for your help

Actually it works in playground but in a web server like apache netbeans it only goes to one of the morphTargets and then it stays there and even though the setInterval function changes the array element it doesn’t change the morph targets to the element obtained in the setInterval function.

Never mind I figured it out

1 Like