I was wondering does anyone know of a way to make a 3d models mouth move based on what vowels are displayed in a text box because I’m trying to make a 3d models mouth move based on the visemes I got from Amazon polly but I’m having trouble figuring out a way to make it so that my 3d models mouth will move based on what is obtained. If you need a visual example I’ve attached one to this topic. And for another example when the value = i the 3d models mouth will move and make a shape when a person says a word with an i vowel in it (or in this case lip syncing).
Do you have all animations for this in your model?
1 Like
Yes I do
If you would be able to create a repro in PG it would be much easier for everybody to help you.
Okay I’ve posted my babylonjs code and the code I use to obtain the visemes
function speakText() {
// Create the JSON parameters for getSynthesizeSpeechUrl
var speechParams = {
OutputFormat: "json",
SampleRate: "16000",
Text: "",
TextType: "text",
VoiceId: "Joanna",
SpeechMarkTypes: ["viseme", "word"]
};
speechParams.Text = document.getElementById("textEntry").value.toLowerCase();
console.log(speechParams.Text);
// Create the Polly service object and presigner object
var polly = new AWS.Polly({apiVersion: '2016-06-10'});
var signer = new AWS.Polly.Presigner(speechParams, polly);
// Create presigned URL of synthesized speech file
signer.getSynthesizeSpeechUrl(speechParams, function(error, url) {
if (error) {
document.getElementById('result').innerHTML = error;
} else {
console.log(url);
console.log(signer);
//var selectedText = "";
var bracket = "}";
var jsonIndex = -1;
var itemHTML = "";
var hash;
//var myJson = [];
fetch(url, headers => 'Content-Type:', 'application/json').then(res => res.text()).then((out) => {
var myJson = "["+out;
console.log(myJson);
//For every index of a end curly bracket } add a comma to the right of it except the last curly bracket.
for(let j = 0; j < myJson.length; j++){
var str = myJson.replace(/(\n)/g,",\n");
console.log(str);
}
//Gets the last index of a comma and removes it
var newJson = str.lastIndexOf(",");
str = str.substring(0, newJson);
finalJson = str+"]";
var jsonParsed = JSON.parse(finalJson);
console.log(jsonParsed);
console.log(jsonParsed[0]['value']);
console.log(jsonParsed.length);
for(var i = 0; i < jsonParsed.length; i++){
viseme = jsonParsed[i]['value'];
if(viseme.includes('e')){console.log("got it");}
if(viseme.includes('a')){console.log("Gimme an a");}
if(viseme.includes('i')){console.log("Here's an e");}
if(viseme.includes('u')){console.log("Found a u");}
if(viseme.includes('o')){console.log("I see an o");}
time = jsonParsed[i]['time'];
startTime = jsonParsed[i]['start'];
endTime = jsonParsed[i]['end'];
console.log(viseme);
console.log(time);
console.log(startTime);
console.log(endTime);
}
}).catch(error => {throw error});
/*document.getElementById('audioSource').src = url;
document.getElementById('audioPlayback').load();
document.getElementById('result').innerHTML = "Speech ready to play.";
document.getElementById('playButton');*/
}
});
}
var createScene = function () {
var scene = new BABYLON.Scene(engine);
//var layer = new BABYLON.Layer(’’, “Cyber_Background.jpg”, scene, true);
//var hdrTexture = BABYLON.CubeTexture.CreateFromPrefilteredData(“textures/environment.dds”, scene);
//var hdrTexture = BABYLON.CubeTexture.CreateFromPrefilteredData(“Cyber_Background.jpg”, scene);
//var currentSkybox = scene.createDefaultSkybox(hdrTexture, true);
//var camera = new BABYLON.FreeCamera(“camera1”, new BABYLON.Vector3(0, 5, -10), scene);
var camera = new BABYLON.ArcRotateCamera(“camera1”, Math.PI / -2, 1, 3, new BABYLON.Vector3(0, 3, 0), scene);
camera.attachControl(canvas, true);
//camera.position = new BABYLON.Vector3(0, 50, -40);
var light = new BABYLON.HemisphericLight(“light1”, new BABYLON.Vector3(0, 1, 0), scene);
light.intensity = 0.7;
var ground = BABYLON.Mesh.CreateGround(“ground1”, 6, 6, 2, scene);
ground.receiveShadows = true;
var gui;
var scales = [];
var url;
var fileName;
url = 'https://raw.githubusercontent.com/chris45242/BabylonModel/main/';
fileName = "project.blend1.gltf";
var Casi = BABYLON.SceneLoader.Append(url, fileName, scene, function (scene){
// Create a default arc rotate camera and light.
scene.createDefaultCameraOrLight(true, true, true);
scene.activeCamera.alpha += Math.PI;
scene.stopAllAnimations();
scene.animationGroups[0].start(true);
scene.animationGroups[1].start(true);
//Get's some of the meshes that are used to make Casi
var casiBody = scene.getMeshByName("Casi's Body.001_primitive0");
var casiInnerMouth = scene.getMeshByName("Casi's Body.001_primitive2");
/*casiInnerMouth.setEnabled(false);
var casiEyes = scene.getMeshByName("Casi's Body.001_primitive1");
var casiTeeth = scene.getMeshByName("Casi's Teeth");*/
var primitive = scene.getMeshByName("Primitives.001");
//Gets Casi's Visor and uses visibility to make it see through glass-like material
var casiVisor = scene.getMeshByName("Casi's Visor");
casiVisor.setEnabled(true);
casiVisor.visibility = 0.7;
//Set up Morph Targets for Casi before the screen is done loading.
let t = 0;
var speak = document.getElementById("speaker").innerHTML;
console.log(speak);
lipSync = scene.onBeforeRenderObservable.add(function(){
//casiBody.morphTargetManager.getTarget(1).influence = Math.abs(Math.sin(t));
primitive.morphTargetManager.getTarget(2).influence = Math.abs(Math.sin(t));
casiBody.morphTargetManager.getTarget(6).influence = Math.abs(Math.sin(t));
//casiBody.morphTargetManager.getTarget(6).influence = 1;
casiInnerMouth.morphTargetManager.getTarget(6).influence = Math.abs(Math.sin(t));
//var getViseme = viseme;
//console.log(getViseme);
//casiInnerMouth.morphTargetManager.getTarget(6).influence = 1;
//casiBody.morphTargetManager.getTarget(0).influence = Math.abs(Math.cos(t));
t += 0.07;
});
});
return scene;
};
var engine = new BABYLON.Engine(canvas, true, { preserveDrawingBuffer: true, stencil: true });
var scene = createScene();
engine.runRenderLoop(function () {
if (scene) {
scene.render();
}
});
Hi, @Chris1 !
I think, custom observables are helpful for your case.
I made a simple PG with it.
I hope it helps!
3 Likes
Hello there! There is an article about that here:
When I tried it the vowelInput.onValueChange = new BABYLON.Observable(); worked but everything else beneath it didn’t. Am I missing something?
Would be great if you could share a repro ?
Do you mean a playground or git repository?
A playground would be great ?
I’m still a little new to BabylonJS do you know how I can post a playground link because I still haven’t figured that out
yup you can go here https://playground.babylonjs.com/ create the repro of the issue, save and copy the new url after saving in here 
Here’s a playground of my problem
Your code should be in the callback https://playground.babylonjs.com/#VU1G98#6
It works thank you for your help
Actually it works in playground but in a web server like apache netbeans it only goes to one of the morphTargets and then it stays there and even though the setInterval function changes the array element it doesn’t change the morph targets to the element obtained in the setInterval function.
Never mind I figured it out
1 Like
Is there any way I can make it so that the time interval in the setInterval function will change at runtime because I’m also trying to make it so that after my 3D model’s lip position is taken the setInterval function will be executed again but with the next time value in the screenshot shown at the top of the page and it will continue to do so until we reach the end of the array
In this case it might be easier to hook into onBeforeRenderObservable and maintain yourself a time counter where you could sync however you want instead of relying on plenty of setInterval ?