Using observables for lip syncing

I was wondering does anyone know of a way to make a 3d models mouth move based on what vowels are displayed in a text box because I’m trying to make a 3d models mouth move based on the visemes I got from Amazon polly but I’m having trouble figuring out a way to make it so that my 3d models mouth will move based on what is obtained. If you need a visual example I’ve attached one to this topic. And for another example when the value = i the 3d models mouth will move and make a shape when a person says a word with an i vowel in it (or in this case lip syncing).
Screenshot 2022-01-01 192211

Do you have all animations for this in your model?

Yes I do

If you would be able to create a repro in PG it would be much easier for everybody to help you.

Okay I’ve posted my babylonjs code and the code I use to obtain the visemes
function speakText() {

        // Create the JSON parameters for getSynthesizeSpeechUrl
        var speechParams = {
            OutputFormat: "json",
            SampleRate: "16000",
            Text: "",
            TextType: "text",
            VoiceId: "Joanna",
            SpeechMarkTypes: ["viseme", "word"]
        };
        speechParams.Text = document.getElementById("textEntry").value.toLowerCase(); 
        console.log(speechParams.Text);
        
        // Create the Polly service object and presigner object
        var polly = new AWS.Polly({apiVersion: '2016-06-10'});
        var signer = new AWS.Polly.Presigner(speechParams, polly);

        // Create presigned URL of synthesized speech file
        signer.getSynthesizeSpeechUrl(speechParams, function(error, url) {
        if (error) {
            document.getElementById('result').innerHTML = error;
        } else {
            
            console.log(url);
            console.log(signer);
            
            //var selectedText = "";
            var bracket = "}";
            var jsonIndex = -1;
            var itemHTML = "";
            var hash;
            //var myJson = [];
            
            fetch(url, headers => 'Content-Type:', 'application/json').then(res => res.text()).then((out) => {
                var myJson = "["+out;
                console.log(myJson);
                //For every index of a end curly bracket } add a comma to the right of it except the last curly bracket.
                for(let j = 0; j < myJson.length; j++){
                    var str = myJson.replace(/(\n)/g,",\n");
                    console.log(str);
                }
                //Gets the last index of a comma and removes it
                var newJson = str.lastIndexOf(",");
                str = str.substring(0, newJson);
                finalJson = str+"]";
               
                var jsonParsed = JSON.parse(finalJson);
                console.log(jsonParsed);
                console.log(jsonParsed[0]['value']);
                console.log(jsonParsed.length);
                for(var i = 0; i < jsonParsed.length; i++){
                    viseme = jsonParsed[i]['value'];
                    if(viseme.includes('e')){console.log("got it");}
                    if(viseme.includes('a')){console.log("Gimme an a");}
                    if(viseme.includes('i')){console.log("Here's an e");}
                    if(viseme.includes('u')){console.log("Found a u");}
                    if(viseme.includes('o')){console.log("I see an o");}
                    time = jsonParsed[i]['time'];
                    startTime = jsonParsed[i]['start'];
                    endTime = jsonParsed[i]['end'];
                    console.log(viseme);
                    console.log(time);
                    console.log(startTime);
                    console.log(endTime);
                    
                }
            }).catch(error => {throw error});
            /*document.getElementById('audioSource').src = url;
            document.getElementById('audioPlayback').load();
            document.getElementById('result').innerHTML = "Speech ready to play.";
            document.getElementById('playButton');*/
        }
      });
        }

var createScene = function () {
var scene = new BABYLON.Scene(engine);
//var layer = new BABYLON.Layer(’’, “Cyber_Background.jpg”, scene, true);
//var hdrTexture = BABYLON.CubeTexture.CreateFromPrefilteredData(“textures/environment.dds”, scene);
//var hdrTexture = BABYLON.CubeTexture.CreateFromPrefilteredData(“Cyber_Background.jpg”, scene);
//var currentSkybox = scene.createDefaultSkybox(hdrTexture, true);
//var camera = new BABYLON.FreeCamera(“camera1”, new BABYLON.Vector3(0, 5, -10), scene);
var camera = new BABYLON.ArcRotateCamera(“camera1”, Math.PI / -2, 1, 3, new BABYLON.Vector3(0, 3, 0), scene);
camera.attachControl(canvas, true);
//camera.position = new BABYLON.Vector3(0, 50, -40);
var light = new BABYLON.HemisphericLight(“light1”, new BABYLON.Vector3(0, 1, 0), scene);
light.intensity = 0.7;
var ground = BABYLON.Mesh.CreateGround(“ground1”, 6, 6, 2, scene);
ground.receiveShadows = true;
var gui;

    var scales = [];
var url;
var fileName;
    
    url = 'https://raw.githubusercontent.com/chris45242/BabylonModel/main/';
    fileName = "project.blend1.gltf";

    var Casi = BABYLON.SceneLoader.Append(url, fileName, scene, function (scene){
            // Create a default arc rotate camera and light.
            scene.createDefaultCameraOrLight(true, true, true);
            scene.activeCamera.alpha += Math.PI;
            
            scene.stopAllAnimations();

	scene.animationGroups[0].start(true); 
            scene.animationGroups[1].start(true);

            //Get's some of the meshes that are used to make Casi
            var casiBody = scene.getMeshByName("Casi's Body.001_primitive0");
            var casiInnerMouth = scene.getMeshByName("Casi's Body.001_primitive2");
            /*casiInnerMouth.setEnabled(false);
            var casiEyes = scene.getMeshByName("Casi's Body.001_primitive1");
            var casiTeeth = scene.getMeshByName("Casi's Teeth");*/
            var primitive = scene.getMeshByName("Primitives.001");
            
            //Gets Casi's Visor and uses visibility to make it see through glass-like material
            var casiVisor = scene.getMeshByName("Casi's Visor");
            casiVisor.setEnabled(true);
            casiVisor.visibility = 0.7;
            
            //Set up Morph Targets for Casi before the screen is done loading.
            let t = 0;
            var speak = document.getElementById("speaker").innerHTML;
            console.log(speak);
            lipSync = scene.onBeforeRenderObservable.add(function(){
                //casiBody.morphTargetManager.getTarget(1).influence = Math.abs(Math.sin(t));
                primitive.morphTargetManager.getTarget(2).influence = Math.abs(Math.sin(t));
                casiBody.morphTargetManager.getTarget(6).influence = Math.abs(Math.sin(t));
                //casiBody.morphTargetManager.getTarget(6).influence = 1;
                casiInnerMouth.morphTargetManager.getTarget(6).influence = Math.abs(Math.sin(t));
                //var getViseme = viseme;
                //console.log(getViseme);
                //casiInnerMouth.morphTargetManager.getTarget(6).influence = 1;
                //casiBody.morphTargetManager.getTarget(0).influence = Math.abs(Math.cos(t));
                t += 0.07;
        });

});
return scene;

};

var engine = new BABYLON.Engine(canvas, true, { preserveDrawingBuffer: true, stencil: true });

var scene = createScene();

engine.runRenderLoop(function () {
if (scene) {
scene.render();
}
});

Hi, @Chris1 !

I think, custom observables are helpful for your case.
I made a simple PG with it.

I hope it helps!

3 Likes