GLSL Tessellation Displacement Mapping

Question

in my recent project I am working with hardware side tessellation. The pipeline I want to implement should take a low poly mesh, tessellate it and apply a displacement map.

The Tessellation works fine and just as I expected it to look like. However, when I apply the displacement map in the tessellation evaluation shader I get an output which is somewhat random.

This is the output without displacement (I used the heightmap as a texture to verify whether my texCoords are accurate)

This is what I get when I enable my displacement (using the same texture for both coloring and displacement):

The shader code is as follows:

//VERTEX SHADER

#version 430

layout(location = 0) in vec4 vertex;
layout(location = 1) in vec4 normal;
layout(location = 2) in vec2 texCoord;

out vec3 vPosition;
out vec3 vNormal;
out vec2 vTexCoord;

void main() {
    vPosition = vertex.xyz;
    vNormal = normal.xyz;
    vTexCoord = texCoord;
}

//TESS CONTROL

#version 430

layout(vertices = 3) out;

in vec3 vPosition[];
in vec3 vNormal[];
in vec2 vTexCoord[];
out vec3 tcPosition[];
out vec3 tcNormal[];
out vec2 tcTexCoord[];

uniform float innerTessLevel;
uniform float outerTessLevel;

void main(){

    float inTess  = innerTessLevel;
    float outTess = outerTessLevel;

    tcPosition[gl_InvocationID] = vPosition[gl_InvocationID];
    tcNormal[gl_InvocationID]   = vNormal[gl_InvocationID];
    tcTexCoord[gl_InvocationID] = vTexCoord[gl_InvocationID];
    if(gl_InvocationID == 0) {
        gl_TessLevelInner[0] = inTess;
        gl_TessLevelInner[1] = inTess;
        gl_TessLevelOuter[0] = outTess;
        gl_TessLevelOuter[1] = outTess;
        gl_TessLevelOuter[2] = outTess;
        gl_TessLevelOuter[3] = outTess;
    }
}

//TESS EVAL

#version 430

layout(triangles, equal_spacing, ccw) in;

in vec3 tcPosition[];
in vec3 tcNormal[];
in vec2 tcTexCoord[];
out vec3 tePosition;
out vec2 teTexCoord;

uniform mat4 ModelViewProjection;
uniform mat4 ModelView;

uniform sampler2D texHeight;

void main(){
    vec3 p0 = gl_TessCoord.x * tcPosition[0];
    vec3 p1 = gl_TessCoord.y * tcPosition[1];
    vec3 p2 = gl_TessCoord.z * tcPosition[2];
    vec3 pos = p0 + p1 + p2;

    vec3 n0 = gl_TessCoord.x * tcNormal[0];
    vec3 n1 = gl_TessCoord.y * tcNormal[1];
    vec3 n2 = gl_TessCoord.z * tcNormal[2];
    vec3 normal = normalize(n0 + n1 + n2);

    vec2 tc0 = gl_TessCoord.x * tcTexCoord[0];
    vec2 tc1 = gl_TessCoord.y * tcTexCoord[1];
    vec2 tc2 = gl_TessCoord.z * tcTexCoord[2];  
    teTexCoord = tc0 + tc1 + tc2;

    float height = texture(texHeight, teTexCoord).x;
    pos += normal * (height * 0.2f);

    gl_Position = ModelViewProjection * vec4(pos, 1);
    tePosition = vec3(ModelView * vec4(pos,1.0)).xyz;
}

//GEOMETRY

#version 430

layout(triangles) in;
layout(triangle_strip, max_vertices = 3) out;

uniform mat4 ModelView;

in vec3 tePosition[3];
in vec3 tePatchDistance[3];
in vec2 teTexCoord[3];
out vec3 gFacetNormal;
out vec2 gTexCoord;

void main() {
    vec3 A = tePosition[2] - tePosition[0];
    vec3 B = tePosition[1] - tePosition[0];
    vec4 N = vec4( normalize(cross(A, B)) , 0.0);
    gFacetNormal = N.xyz;

    gTexCoord = teTexCoord[0];
    gl_Position = gl_in[0].gl_Position; EmitVertex();

    gTexCoord = teTexCoord[1];
    gl_Position = gl_in[1].gl_Position; EmitVertex();

    gTexCoord = teTexCoord[2];
    gl_Position = gl_in[2].gl_Position; EmitVertex();

    EndPrimitive();
}

//FRAGMENT

#version 430

layout(location = 0) out vec4 fragColor;

in vec3 gFacetNormal;
in vec2 gTexCoord;

uniform float lit;
uniform vec3 light;
uniform sampler2D texHeight;

void main() {
    #ifndef ORANGE_PURPLE
        vec3 color = gl_FrontFacing ? vec3(1.0,0.0,0.0) : vec3(0.0,0.0,1.0);
    #else
        vec3 color = gl_FrontFacing ? vec3(1.0,0.6,0.0) : vec3(0.6,0.0,1.0);
    #endif
    if (lit > 0.5) {
        color = texture(texHeight, gTexCoord).xyz; 
        vec3 N = normalize(gFacetNormal);
        vec3 L = light;
        float df = abs(dot(N,L));
        color = df * color;

        fragColor = vec4(color,1.0);
    }
    else {
        fragColor = vec4(color,1.0);
    }
}

It would be nice if someone could help me on that one.

Baseless

Should not displacement always be facing outward from the surface given a unorm texture (right now you have some areas where the displacement appears to move the vertices inside the cylinder)? I strongly suspect there is something wrong with your normal calculation in the TES. Perhaps you could pass the normal you calculate in the TES through to geometry/fragment and visualize it to see if it looks correct? – Fortna

In fact: the output gFacetNormal is only defined for your first vertex in the geometry shader. Outputs have to be set after every EmitVertex (...) as-per the GLSL specification, or they will be undefined. Many implementations re-use the last value set, but you cannot rely on that behavior if you want this to work portably. You need to set gFacetNormal once before every EmitVertex. void EmitVertex () - "Emits the current values of output variables to the current output primitive. On return from this call, the values of output variables are undefined." – Fortna

Thanks for the advice!!! I wouldn't have thought this would do the trick, but it was really actually the ´gFacetNormal´ that busted everything. I'll edit the code for everybody else who is interested in implementing the routine. – Baseless

Answer 1

Thanks to @AndonM.Coleman I solved the matter

In fact: the output gFacetNormal is only defined for your first vertex in the geometry shader. Outputs have to be set after every EmitVertex (...) as-per the GLSL specification, or they will be undefined. Many implementations re-use the last value set, but you cannot rely on that behavior if you want this to work portably. You need to set gFacetNormal once before every EmitVertex. void EmitVertex () - "Emits the current values of output variables to the current output primitive. On return from this call, the values of output variables are undefined."

Stupid of me not to notice that, but here is the working code:

//VERTEX

#version 430

layout(location = 0) in vec4 vertex;
layout(location = 1) in vec4 normal;
layout(location = 2) in vec2 texCoord;

out vec3 vPosition;
out vec3 vNormal;
out vec2 vTexCoord;

void main() {
    vPosition = vertex.xyz;
    vNormal = normal.xyz;
    vTexCoord = texCoord;
}

//TESSELLATION CONTROL

#version 430

layout(vertices = 3) out;

in vec3 vPosition[];
in vec3 vNormal[];
in vec2 vTexCoord[];
out vec3 tcPosition[];
out vec3 tcNormal[];
out vec2 tcTexCoord[];

uniform float innerTessLevel;
uniform float outerTessLevel;

void main(){

    float inTess  = innerTessLevel;
    float outTess = outerTessLevel;

    tcPosition[gl_InvocationID] = vPosition[gl_InvocationID];
    tcNormal[gl_InvocationID]   = vNormal[gl_InvocationID];
    tcTexCoord[gl_InvocationID] = vTexCoord[gl_InvocationID];
    if(gl_InvocationID == 0) {
        gl_TessLevelInner[0] = inTess;
        gl_TessLevelInner[1] = inTess;
        gl_TessLevelOuter[0] = outTess;
        gl_TessLevelOuter[1] = outTess;
        gl_TessLevelOuter[2] = outTess;
        gl_TessLevelOuter[3] = outTess;
    }
}

//TESSELLATION EVALUATION

#version 430

layout(triangles, equal_spacing, ccw) in;

in vec3 tcPosition[];
in vec3 tcNormal[];
in vec2 tcTexCoord[];
out vec3 tePosition;
out vec2 teTexCoord;
out vec3 teNormal;

uniform mat4 ModelViewProjection;
uniform mat4 ModelView;

uniform sampler2D texHeight;

void main(){
    vec3 p0 = gl_TessCoord.x * tcPosition[0];
    vec3 p1 = gl_TessCoord.y * tcPosition[1];
    vec3 p2 = gl_TessCoord.z * tcPosition[2];
    vec3 pos = p0 + p1 + p2;

    vec3 n0 = gl_TessCoord.x * tcNormal[0];
    vec3 n1 = gl_TessCoord.y * tcNormal[1];
    vec3 n2 = gl_TessCoord.z * tcNormal[2];
    vec3 normal = normalize(n0 + n1 + n2);

    vec2 tc0 = gl_TessCoord.x * tcTexCoord[0];
    vec2 tc1 = gl_TessCoord.y * tcTexCoord[1];
    vec2 tc2 = gl_TessCoord.z * tcTexCoord[2];  
    teTexCoord = tc0 + tc1 + tc2;

    float height = texture(texHeight, teTexCoord).x;
    pos += normal * (height * 0.5f);

    gl_Position = ModelViewProjection * vec4(pos, 1);
    teNormal    = vec3(ModelView * vec4(normal,0.0)).xyz;
    tePosition  = vec3(ModelView * vec4(pos,1.0)).xyz;
}

//GEOMETRY

#version 430

layout(triangles) in;
layout(triangle_strip, max_vertices = 3) out;

uniform mat4 ModelView;

in vec3 tePosition[3];
in vec2 teTexCoord[3];
in vec3 teNormal[3];
out vec3 gFacetNormal;
out vec2 gTexCoord;

void main() {

    gFacetNormal = teNormal[0];
    gTexCoord = teTexCoord[0];
    gl_Position = gl_in[0].gl_Position; EmitVertex();

    gFacetNormal = teNormal[1];
    gTexCoord = teTexCoord[1];
    gl_Position = gl_in[1].gl_Position; EmitVertex();

    gFacetNormal = teNormal[2];
    gTexCoord = teTexCoord[2];
    gl_Position = gl_in[2].gl_Position; EmitVertex();

    EndPrimitive();
}

//FRAGMENT

#version 430

layout(location = 0) out vec4 fragColor;

in vec3 gFacetNormal;
in vec2 gTexCoord;

uniform float lit;
uniform vec3 light;
uniform sampler2D texHeight;

void main() {
    #ifndef ORANGE_PURPLE
    vec3 color = gl_FrontFacing ? vec3(1.0,0.0,0.0) : vec3(0.0,0.0,1.0);
    #else
        vec3 color = gl_FrontFacing ? vec3(1.0,0.6,0.0) : vec3(0.6,0.0,1.0);
    #endif
    if (lit > 0.5) {
        color = texture(texHeight, gTexCoord).xyz; 
        vec3 N = normalize(gFacetNormal);
        vec3 L = light;
        float df = abs(dot(N,L));
        color = df * color;
        fragColor = vec4(color,1.0);
    }
    else {
        fragColor = vec4(color,1.0);
    }
}

Baseless

To be honest, I am a little bit confused. Is this the only thing you changed? The Geometry Shader runs after tessellation, so the really weird displacement should not be solved by making a modification to the GS. What it might solve though are issues with lighting. – Fortna

Well like I commented on your advise I was pretty confused, too. But the shader code is the only thing I edited. Concerning these changes I only modified the per vertex output and since I am passing the normals through ALL stages anyway I deleted the calculation of normals in the geometry shader. – Baseless