Vertex animation texture with material plugin

Questions
, ,
1

hello there

i’m currently working on custom vertex animation texture
the process is like this

  1. import animated 3d models into bledner
  2. run custom python script that export glb file and bake animation on png file
  3. import glb file and texture into babylon
  4. run animation with help of GPU “vertex shader”

at begin i do all of that with shader material
and work very well in amination section

import { Effect, ShaderMaterial, Texture } from "@babylonjs/core";
import Editor from "./Editor";

export default class VatMaterial extends ShaderMaterial {
    constructor(name: string, textureName: string) {
        const { scene, assetManager } = Editor.GetInstance()
        super(name, scene, "vat", {
            attributes: ["position", "normal", "uv", "indexX", "minMaxX", "minMaxY", "minMaxZ"],
            defines: ["#define INSTANCES"],
            uniforms: ["worldViewProjection", "posTex", "frame"]
        });

        Effect.ShadersStore.vatVertexShader = `
                precision highp float;

                attribute vec3 position;
                attribute vec2 uv;

                attribute float indexX;
                attribute vec2 minMaxX;
                attribute vec2 minMaxY;
                attribute vec2 minMaxZ;

                uniform mat4 worldViewProjection;
                uniform sampler2D posTex;
                uniform float frame;

                uniform float posTexWidth;
                uniform float posTexHeight;

                void main() {
                    vec4 texturePos = texture2D(posTex,vec2(indexX, 1.0 - frame));

                    float minX = minMaxX.x;
                    float maxX = minMaxX.y;

                    float minY = minMaxY.x;
                    float maxY = minMaxY.y;

                    float minZ = minMaxZ.x;
                    float maxZ = minMaxZ.y;

                    float posX = texturePos.x * (maxX - minX) + minX;
                    float posY = texturePos.y * (maxY - minY) + minY;
                    float posZ = texturePos.z * (maxZ - minZ) + minZ;

                    // gl_Position = worldViewProjection * vec4(position.x,position.y,position.z, 1.0);
                    gl_Position = worldViewProjection * vec4(posX,posY,posZ, 1.0);
            }`;


        Effect.ShadersStore.vatPixelShader = `
                precision highp float;

                void main(void) {
                   gl_FragColor = vec4(1.0,0.0,0.0,1.0);
                }`;


        const posTexture = assetManager.getTexture(textureName)
        posTexture.wrapU = Texture.WRAP_ADDRESSMODE;
        posTexture.wrapV = Texture.WRAP_ADDRESSMODE;

        const size = posTexture.getSize()

        this.setTexture("posTex", posTexture);
        this.setFloat("posTexWidth", size.width);
        this.setFloat("posTexHeight", size.height);

        let a = 0
        scene.onBeforeRenderObservable.add(() => {
            a += 0.002
            if (a > 1) a = 0.0
            this.setFloat("frame", a)
        })
    }
}

but material do not render good picture
because of fragment shader
after some research i find out
i can customize default materials with
help of “Material Plugin”

import { Material, MaterialDefines, MaterialPluginBase, ShaderLanguage, Texture, UniformBuffer } from "@babylonjs/core";

export default class VatMaterialPlugin extends MaterialPluginBase {
    texture: Texture
    textureWidth: number
    textureHeight: number
    frame: number

    constructor(material: Material, texture: Texture) {
        super(material, "VatMaterialPlugin", 200, { VAT: false });
        this.texture = texture;

        this.texture.wrapU = Texture.WRAP_ADDRESSMODE;
        this.texture.wrapV = Texture.WRAP_ADDRESSMODE;

        const size = this.texture.getSize()
        this.textureWidth = size.width
        this.textureHeight = size.height
        this.frame = 0.0

        this._enable(true);
    }

    prepareDefines(defines: MaterialDefines) {
        defines["VAT"] = true;
    }

    getClassName() {
        return "VatMaterialPlugin";
    }

    getSamplers(samplers: string[]) {
        samplers.push("posTex", "posTexWidth", "posTexHeight", "frame")
    }

    getAttributes(attributes: string[]) {
        attributes.push("indexX", "minMaxX", "minMaxY", "minMaxZ")
    }

    bindForSubMesh(uniformBuffer: UniformBuffer): void {
        uniformBuffer.setTexture("posTex", this.texture);
        uniformBuffer.updateFloat("posTexWidth", this.textureWidth);
        uniformBuffer.updateFloat("posTexHeight", this.textureHeight);
        uniformBuffer.updateFloat("frame", this.frame)
    }

    getUniforms() {
        return {
            "ubo": [
                { name: "posTexWidth", size: 1, type: "float" },
                { name: "posTexHeight", size: 1, type: "float" },
                { name: "frame", size: 1, type: "float" },
            ],
            "vertex":
                `#ifdef VAT
                    uniform float posTexWidth;
                    uniform float posTexHeight;
                    uniform float frame;
                #endif
                `,
        }
    }

    // This is used to inform the system which language is supported
    isCompatible(shaderLanguage: ShaderLanguage) {
        switch (shaderLanguage) {
            case ShaderLanguage.GLSL:
                return true;
            case ShaderLanguage.WGSL:
            default:
                return false;
        }
    }

    getCustomCode(shaderType: string, shaderLanguage: ShaderLanguage) {
        if (shaderType === "vertex") {
            if (shaderLanguage === ShaderLanguage.GLSL) {
                return {
                    "CUSTOM_VERTEX_DEFINITIONS": `
                    attribute float indexX;
                    attribute vec2 minMaxX;
                    attribute vec2 minMaxY;
                    attribute vec2 minMaxZ;

                    uniform mat4 worldViewProjection;
                    uniform sampler2D posTex;
                `,
                    "CUSTOM_VERTEX_MAIN_END": `
                    vec4 texturePos = texture2D(posTex,vec2(indexX, 1.0 - frame));

                    float minX = minMaxX.x;
                    float maxX = minMaxX.y;

                    float minY = minMaxY.x;
                    float maxY = minMaxY.y;

                    float minZ = minMaxZ.x;
                    float maxZ = minMaxZ.y;

                    float posX = texturePos.x * (maxX - minX) + minX;
                    float posY = texturePos.y * (maxY - minY) + minY;
                    float posZ = texturePos.z * (maxZ - minZ) + minZ;

                    gl_Position = worldViewProjection * vec4(posX,posY,posZ, 1.0);
                    `
                }
            }
        }
        return null;
    }
}

but is not working at all
i think problem is with my “worldViewProjection”
is that valid way to import that like this “uniform mat4 worldViewProjection;”
or should I passed some way to my material plugin?

source code is available on github

thanks for helping

2

Material plugin add code to the shader, so you can use view and viewprojection in pbr vertex for example, as it is defined in pbrVertexDeclaration.fx.
For worldViewProjection, it is not defined in standard material nor pbr material, so you should pass it on your own like other uniforms (frame in your code for example).

1 Like
3

i test that “viewProjection” is not showing anything
is that same as “worldViewProjection” ?

you mention defined “worldViewProjection” and pass it.
in shader material is default uniform i think.
how can i calculate “worldViewProjection” ?

4

thats work

viewProjection * finalWorld work as worldViewProjection

thanks for your help

5

it was not necessary at all i find out in source there is section that call CUSTOM_VERTEX_UPDATE_POSITION and directly update positions in here.

thats work fine even i got PBRMaterial effect on my cloth
but is rotation on camera rotation ( its like double camera effect )

should i bake normal in blender to?
im just forgot to do somthing?

final code is like this

import { Material, MaterialDefines, MaterialPluginBase, ShaderLanguage, Texture, UniformBuffer } from "@babylonjs/core";

export default class VatMaterialPlugin extends MaterialPluginBase {
    texture: Texture
    frame: number

    constructor(material: Material, texture: Texture) {
        super(material, "VatMaterialPlugin", 200, { VAT: false });
        this.texture = texture;

        this.texture.wrapU = Texture.WRAP_ADDRESSMODE;
        this.texture.wrapV = Texture.WRAP_ADDRESSMODE;

        this.frame = 0.0

        this._enable(true);
    }

    prepareDefines(defines: MaterialDefines) {
        defines["VAT"] = true;
    }

    getClassName() {
        return "VatMaterialPlugin";
    }

    getSamplers(samplers: string[]) {
        samplers.push("posTex", "posTexWidth", "posTexHeight", "frame")
    }

    getAttributes(attributes: string[]) {
        attributes.push("indexX", "minMaxX", "minMaxY", "minMaxZ")
    }

    bindForSubMesh(uniformBuffer: UniformBuffer): void {
        uniformBuffer.setTexture("posTex", this.texture);
        uniformBuffer.updateFloat("frame", this.frame)
    }

    getUniforms() {
        return {
            "ubo": [
                { name: "frame", size: 1, type: "float" },
            ],
            "vertex":
                `#ifdef VAT
                    uniform float frame;
                #endif
                `,
        }
    }

    // This is used to inform the system which language is supported
    isCompatible(shaderLanguage: ShaderLanguage) {
        switch (shaderLanguage) {
            case ShaderLanguage.GLSL:
                return true;
            case ShaderLanguage.WGSL:
            default:
                return false;
        }
    }

    getCustomCode(shaderType: string, shaderLanguage: ShaderLanguage) {
        if (shaderType === "vertex") {
            if (shaderLanguage === ShaderLanguage.GLSL) {
                return {
                    "CUSTOM_VERTEX_DEFINITIONS": `
                    attribute float indexX;
                    attribute vec2 minMaxX;
                    attribute vec2 minMaxY;
                    attribute vec2 minMaxZ;

                    // uniform mat4 worldViewProjection;
                    uniform sampler2D posTex;
                `,
                    "CUSTOM_VERTEX_UPDATE_POSITION": `
                    vec4 texturePos = texture2D(posTex,vec2(indexX, 1.0 - frame));

                    float minX = minMaxX.x;
                    float maxX = minMaxX.y;

                    float minY = minMaxY.x;
                    float maxY = minMaxY.y;

                    float minZ = minMaxZ.x;
                    float maxZ = minMaxZ.y;

                    float posX = texturePos.x * (maxX - minX) + minX;
                    float posY = texturePos.y * (maxY - minY) + minY;
                    float posZ = texturePos.z * (maxZ - minZ) + minZ;

                    // positionUpdated = vec3(posX,posY,posZ);
                    positionUpdated = vec3(posX,posY,posZ);
                    // gl_Position = viewProjection * worldPos * vec4(posX,posY,posZ, 1.0);
                    `
                }
            }
        }
        return null;
    }
}
6

yep!
that was the problem I should also bake normal and process in “CUSTOM_VERTEX_UPDATE_NORMAL” section

final material plugin is look like this

import { Material, MaterialDefines, MaterialPluginBase, ShaderLanguage, Texture, UniformBuffer } from "@babylonjs/core";

export default class VatMaterialPlugin extends MaterialPluginBase {
    texture: Texture
    normalTexture: Texture
    frame: number

    constructor(material: Material, texture: Texture, normalTexture: Texture) {
        super(material, "VatMaterialPlugin", 200, { VAT: false });
        this.texture = texture;
        this.normalTexture = normalTexture;

        this.texture.wrapU = Texture.WRAP_ADDRESSMODE;
        this.texture.wrapV = Texture.WRAP_ADDRESSMODE;

        this.frame = 0.0

        this._enable(true);
    }

    prepareDefines(defines: MaterialDefines) {
        defines["VAT"] = true;
    }

    getClassName() {
        return "VatMaterialPlugin";
    }

    getSamplers(samplers: string[]) {
        samplers.push("posTex", "norTex", "frame")
    }

    getAttributes(attributes: string[]) {
        attributes.push("indexX", "minMaxX", "minMaxY", "minMaxZ", "minMaxNX", "minMaxNY", "minMaxNZ")
    }

    bindForSubMesh(uniformBuffer: UniformBuffer): void {
        uniformBuffer.setTexture("posTex", this.texture);
        uniformBuffer.setTexture("norTex", this.normalTexture);
        uniformBuffer.updateFloat("frame", this.frame)
    }

    getUniforms() {
        return {
            "ubo": [
                { name: "frame", size: 1, type: "float" },
            ],
            "vertex":
                `#ifdef VAT
                    uniform float frame;
                #endif
                `,
        }
    }

    isCompatible(shaderLanguage: ShaderLanguage) {
        switch (shaderLanguage) {
            case ShaderLanguage.GLSL:
                return true;
            case ShaderLanguage.WGSL:
            default:
                return false;
        }
    }

    getCustomCode(shaderType: string, shaderLanguage: ShaderLanguage) {
        if (shaderType === "vertex") {
            if (shaderLanguage === ShaderLanguage.GLSL) {
                return {
                    "CUSTOM_VERTEX_DEFINITIONS": `
                    attribute float indexX;

                    attribute vec2 minMaxX;
                    attribute vec2 minMaxY;
                    attribute vec2 minMaxZ;

                    attribute vec2 minMaxNX;
                    attribute vec2 minMaxNY;
                    attribute vec2 minMaxNZ;

                    uniform sampler2D posTex;
                    uniform sampler2D norTex;
                `,
                    "CUSTOM_VERTEX_UPDATE_POSITION": `
                    vec4 texturePos = texture2D(posTex,vec2(indexX, 1.0 - frame));


                    float posX = texturePos.x * (minMaxX.y - minMaxX.x) + minMaxX.x;
                    float posY = texturePos.y * (minMaxY.y - minMaxY.x) + minMaxY.x;
                    float posZ = texturePos.z * (minMaxZ.y - minMaxZ.x) + minMaxZ.x;

                    positionUpdated = vec3(posX,posY,posZ);
                    `,
                    "CUSTOM_VERTEX_UPDATE_NORMAL": `
                    vec4 textureNor = texture2D(norTex,vec2(indexX, 1.0 - frame));

                    float norX = textureNor.x * (minMaxNX.y - minMaxNX.x) + minMaxNX.x;
                    float norY = textureNor.y * (minMaxNY.y - minMaxNY.x) + minMaxNY.x;
                    float norZ = textureNor.z * (minMaxNZ.y - minMaxNZ.x) + minMaxNZ.x;

                    normalUpdated = vec3(norX,norY,norZ);
                    `
                }
            }
        }
        return null;
    }
}

and small demo of result
source is also available in github