openglglslshadercullingtessellation

Shader Flipping Faces


I'm trying to construct a render engine using OpenGL and C++. but can't seem to get past this problem. The same model is being rendered 5 different times using different shaders, in 4 out of the 5 shaders the backface culling is working properly. In the tessellation shader, however, it is not. Any outwards faces are invisible, so you can see directly to the rear ones. Does anyone know why this shader flips the faces?

Vertex Shader

void main()
{
    worldVertexPosition_cs = (transformationMatrix * vec4(position_vs, 1.0)).xyz;
    worldTextureCoords_cs = textureCoords_vs;
    worldNormal_cs = mat3(transpose(inverse(transformationMatrix))) * normal_vs;
}

Control Shader

float getTessLevel(float distance0, float distance1)
{
    float avgDistance = (distance0 + distance1) / 2.0;
    avgDistance = (100 - avgDistance) / 20;
    if (avgDistance < 1) {
        avgDistance = 1;
    }
    return avgDistance;
}

void main()
{
    worldTextureCoords_es[gl_InvocationID] = worldTextureCoords_cs[gl_InvocationID];
    worldNormal_es[gl_InvocationID] = worldNormal_cs[gl_InvocationID];
    worldVertexPosition_es[gl_InvocationID] = worldVertexPosition_cs[gl_InvocationID];

    float eyeToVertexDistance0 = distance(eyePos, worldVertexPosition_es[0]);
    float eyeToVertexDistance1 = distance(eyePos, worldVertexPosition_es[1]);
    float eyeToVertexDistance2 = distance(eyePos, worldVertexPosition_es[2]);

    gl_TessLevelOuter[0] = getTessLevel(eyeToVertexDistance1, eyeToVertexDistance2);
    gl_TessLevelOuter[1] = getTessLevel(eyeToVertexDistance2, eyeToVertexDistance0);
    gl_TessLevelOuter[2] = getTessLevel(eyeToVertexDistance0, eyeToVertexDistance1);
    gl_TessLevelInner[0] = gl_TessLevelOuter[2];
}

Evaluation Shader

vec2 interpolate2D(vec2 v0, vec2 v1, vec2 v2)
{
    return vec2(gl_TessCoord.x) * v0 + vec2(gl_TessCoord.y) * v1 + vec2(gl_TessCoord.z) * v2;
}

vec3 interpolate3D(vec3 v0, vec3 v1, vec3 v2)
{
    return vec3(gl_TessCoord.x) * v0 + vec3(gl_TessCoord.y) * v1 + vec3(gl_TessCoord.z) * v2;
}

void main()
{
    worldTextureCoords_fs = interpolate2D(worldTextureCoords_es[0], worldTextureCoords_es[1],         worldTextureCoords_es[2]);
    worldNormal_fs = interpolate3D(worldNormal_es[0], worldNormal_es[1], worldNormal_es[2]);
    worldNormal_fs = normalize(worldNormal_fs);
    worldVertexPosition_fs = interpolate3D(worldVertexPosition_es[0], worldVertexPosition_es[1],     worldVertexPosition_es[2]);

    float displacement = texture(texture_displacement0, worldTextureCoords_fs.xy).x;
    worldVertexPosition_fs += worldNormal_fs * (displacement / 1.0f);

    gl_Position = projectionMatrix * viewMatrix * vec4(worldVertexPosition_fs.xyz, 1.0);
}

Fragment Shader

void main()
{
    vec3 unitNormal = normalize(worldNormal_fs);
    vec3 unitLightVector = normalize(lightPosition - worldVertexPosition_fs);

    float dotResult = dot(unitNormal, unitLightVector);
    float brightness = max(dotResult, blackPoint);
    vec3 diffuse = brightness * lightColor;

    FragColor = vec4(diffuse, 1.0) * texture(texture_diffuse0, worldTextureCoords_fs);
    FragColor.rgb = pow(FragColor.rgb, vec3(1.0/gamma));
}

Solution

  • In the Tessellation Evaluation Shader you've to define the winding order of the generated triangles.
    This is done via the cw and ccw parameters. Default is ccw.

    Either generate clockwise primitives:

    layout(triangles, cw) in;
    

    Or generate counterclockwise primitives:

    layout(triangles, ccw) in;