currently I am working on a OSG Project for my study and wrote a CelShading shader (alongside a simpleFog Shader). I first render with the CelShader along with the depth buffer to Texture and then use the fogShader. Everything works fine on my AMD Radeon HD 7950 and on my Intel HD4400 (although it is slow on the last), both running Windows. However, on a Quadro 600 runnning Linux, the Shader compiles without error, but is still wrong, the light is dulled and because of the lack of some light spots, it seems that not every light in the Scene is used. The whole toon effect is also gone. I confirmed the Shader working on another AMD, a ATI Mobility HD3400.
But on other NVIDIAs, like a GTX 670 or 660 TI oder 560 TI (this time windows) the Shader is not working. First it was totally messed up because of non-uniform flow, but after I fixed it it is still not working.
I have this Problem now for some days and it is giving me a headache. I do not know what am I missing, why is it working on a simple Intel HD 4400 but not on high end NVIDIA Cards? Strangely, the fogShader is working perfectly on every system and gives me the nice fog I want. Does anyone have an idea? The Uniforms are set for the toonTex, but texture0 is not set, because the model is uv-mapped with blender, but the textures seem to work just fine (look at the Pony in the Screens). I assuming 0 is used as layout for texture0, which is perfectly valid,as far as I know. Here is a Video showing the shader on a GTX 660 TI. Something seems to work, if there is only one light, but it is not how it should look like, on a Radeon HD 7950 it is like this (ignore the black border, screenshot issue).
The light is cleary different.
EDIT: Just did another test: on the Intel HD 4400 and Windows, it is working. But the same System running Linux is showing only a whole lot of White with some outlines but no textures at all. Anyone any suggestion? The sources for the shaders are here:
celShader.vert
#version 120
varying vec3 normalModelView;
varying vec4 vertexModelView;
uniform bool zAnimation;
uniform float osg_FrameTime;
void main()
{
normalModelView = gl_NormalMatrix * gl_Normal;
vertexModelView = gl_ModelViewMatrix * gl_Vertex;
gl_TexCoord[0] = gl_MultiTexCoord0;
vec4 vertexPos = gl_Vertex;
if(zAnimation){//
vertexPos.z = sin(5.0*vertexPos.z + osg_FrameTime)*0.25;//+ vertexPos.z;
}
gl_Position = gl_ModelViewProjectionMatrix * vertexPos;
}
celShader.frag
#version 120
#define NUM_LIGHTS 5
uniform sampler2D texture0;
uniform sampler2D toonTex;
uniform float osg_FrameTime;
uniform bool tex;
varying vec3 normalModelView;
varying vec4 vertexModelView;
vec4 calculateLightFromLightSource(int lightIndex, bool front){
vec3 lightDir;
vec3 eye = normalize(-vertexModelView.xyz);
vec4 curLightPos = gl_LightSource[lightIndex].position;
//curLightPos.z = sin(10*osg_FrameTime)*4+curLightPos.z;
lightDir = normalize(curLightPos.xyz - vertexModelView.xyz);
float dist = distance( gl_LightSource[lightIndex].position, vertexModelView );
float attenuation = 1.0 / (gl_LightSource[lightIndex].constantAttenuation
+ gl_LightSource[lightIndex].linearAttenuation * dist
+ gl_LightSource[lightIndex].quadraticAttenuation * dist * dist);
float z = length(vertexModelView);
vec4 color;
vec3 n = normalize(normalModelView);
vec3 nBack = normalize(-normalModelView);
float intensity = dot(n,lightDir); //NdotL, Lambert
float intensityBack = dot(nBack,lightDir); //NdotL, Lambert
//-Phong Modell
vec3 reflected = normalize(reflect( -lightDir, n));
float specular = pow(max(dot(reflected, eye), 0.0), gl_FrontMaterial.shininess);
vec3 reflectedBack = normalize(reflect( -lightDir, nBack));
float specularBack = pow(max(dot(reflectedBack, eye), 0.0), gl_BackMaterial.shininess);
//Toon-Shading
//2D Toon http://www.cs.rpi.edu/~cutler/classes/advancedgraphics/S12/final_projects/hutchins_kim.pdf
vec4 toonColor = texture2D(toonTex,vec2(intensity,specular));
vec4 toonColorBack = texture2D(toonTex,vec2(intensityBack,specularBack));
if(front){
color += gl_FrontMaterial.ambient * gl_LightSource[lightIndex].ambient[lightIndex];
if(intensity > 0.0){
color += gl_FrontMaterial.diffuse * gl_LightSource[lightIndex].diffuse * intensity * attenuation ;
color += gl_FrontMaterial.specular * gl_LightSource[lightIndex].specular * specular *attenuation ;
}
return color * toonColor;
} else {//back
color += gl_BackMaterial.ambient * gl_LightSource[lightIndex].ambient[lightIndex];
if(intensity > 0.0){
color += gl_BackMaterial.diffuse * gl_LightSource[lightIndex].diffuse * intensityBack * attenuation ;
color += gl_BackMaterial.specular * gl_LightSource[lightIndex].specular * specularBack *attenuation ;
}
return color * toonColorBack;
}
}
void main(void) {
vec4 color = vec4(0.0);
bool front = true;
//non-uniform-flow error correction
//see more here: http://www.opengl.org/wiki/GLSL_Sampler#Non-uniform_flow_control
//and here: http://gamedev.stackexchange.com/questions/32543/glsl-if-else-statement-unexpected-behaviour
vec4 texColor = texture2D(texture0,gl_TexCoord[0].xy);
if(!gl_FrontFacing)
front = false;
for(int i = 0; i< NUM_LIGHTS; i++){
color += calculateLightFromLightSource(i,front);
}
if(tex)
gl_FragColor =color * texColor;
else
gl_FragColor = color;
}
fogShader.vert
#version 120
varying vec4 vertexModelView;
void main()
{
gl_Position = ftransform();
vertexModelView = gl_ModelViewMatrix * gl_Vertex;
gl_TexCoord[0] = gl_MultiTexCoord0;
}
fogShader.frag
varying vec4 vertexModelView;
uniform sampler2D texture0;
uniform sampler2D deepth;
uniform vec3 fogColor;
uniform float zNear;
uniform float zFar;
float linearDepth(float z){
return (2.0 * (zNear+zFar)) / ((zFar + zNear) - z * (zFar - zNear));// -1.0;
}
void main(void){
//Literature
//http://www.ozone3d.net/tutorials/glsl_fog/p04.php and depth_of_field example OSG Cookbook
vec2 deepthPoint = gl_TexCoord[0].xy;
float z = texture2D(deepth, deepthPoint).x;
//fogFactor = (end - z) / (end - start)
z = linearDepth(z);
float fogFactor = (4000*4-z) / (4000*4 - 30*4);
fogFactor = clamp(fogFactor, 0.0, 1.0);
vec4 texColor = texture2D(texture0,gl_TexCoord[0].xy);
gl_FragColor = mix(vec4(fogColor,1.0), texColor,fogFactor);
}
ProgramLinking
osg::ref_ptr<osg::Shader> toonFrag = osgDB::readShaderFile("../Shader/celShader.frag");
osg::ref_ptr<osg::Shader> toonVert = osgDB::readShaderFile("../Shader/" + _vertSource);
osg::ref_ptr<osg::Program> celShadingProgram = new osg::Program;
celShadingProgram->addShader(toonFrag);
celShadingProgram->addShader(toonVert);
osg::ref_ptr<osg::Texture2D> toonTex = new osg::Texture2D;
toonTex->setImage(osgDB::readImageFile("../BlenderFiles/Texturen/toons/" + _toonTex));
toonTex->setFilter(osg::Texture::MIN_FILTER, osg::Texture::NEAREST);
toonTex->setFilter(osg::Texture::MAG_FILTER, osg::Texture::NEAREST);
osg::ref_ptr<osg::StateSet> ss = new osg::StateSet;
ss->setTextureAttributeAndModes(1, toonTex, osg::StateAttribute::OVERRIDE | osg::StateAttribute::ON);
ss->addUniform(new osg::Uniform("toonTex", 1));
ss->setAttributeAndModes(celShadingProgram, osg::StateAttribute::OVERRIDE | osg::StateAttribute::ON);
//TODO NEEED?
ss->setTextureMode(1, GL_TEXTURE_1D, osg::StateAttribute::OVERRIDE | osg::StateAttribute::OFF);
ss->addUniform(new osg::Uniform("tex", true));
ss->addUniform(new osg::Uniform("zAnimation", false));
These two lines were broken:
color += gl_FrontMaterial.ambient * gl_LightSource[lightIndex].ambient[lightIndex];
color += gl_BackMaterial.ambient * gl_LightSource[lightIndex].ambient[lightIndex];
The variable ambient is not an array. The actual problem was first pointed out by Reto Koradi in a comment:
ambientis avec4, and you can use array index notation to access vector components. But since you were accessing it with indices up to 4, this part of the spec comes into play: "When indexing with non-constant expressions, behavior is undefined if the index is negative, or greater than or equal to the size of the vector."