What's wrong with my normal mapping? I think it's my tangents

Question

edit: you might want to start at "Edit 3" because I've solved a lot of this

Here's a screenshot of my normal cubemap applied to an icosphere:

The tangents for my cubemapped icosphere are generated with the following code. m_indices in an std::vector of indices into the std::vector of vertices in m_vertices.

std::vector<glm::vec3> storedTan(m_vertices.size(),glm::vec3(0,0,0));

// tangents
for(int i = 0; i < m_indices.size(); i+=3)
{
    int i1 = m_indices[i];
    int i2 = m_indices[i+1];
    int i3 = m_indices[i+2];

    VertexData v1 = m_vertices[i1];
    VertexData v2 = m_vertices[i2];
    VertexData v3 = m_vertices[i3];

    glm::vec3 p1 = glm::vec3(v1.position[0],v1.position[1],v1.position[2]);
    glm::vec3 p2 = glm::vec3(v2.position[0],v2.position[1],v2.position[2]);
    glm::vec3 p3 = glm::vec3(v3.position[0],v3.position[1],v3.position[2]);

    glm::vec3 t1 = glm::vec3(v1.tcoords[0],v1.tcoords[1],v1.tcoords[2]);
    glm::vec3 t2 = glm::vec3(v2.tcoords[0],v2.tcoords[1],v2.tcoords[2]);
    glm::vec3 t3 = glm::vec3(v3.tcoords[0],v3.tcoords[1],v3.tcoords[2]);

    std::function<glm::vec2(glm::vec3)> get_uv = [=](glm::vec3 STR)
    {
        float sc, tc, ma;
        float x = std::abs(STR.x);
        float y = std::abs(STR.y);
        float z = std::abs(STR.z);
        if(x > y && x > z)
        {
            if(STR.x > 0)
            {
                sc = -STR.z;
                tc = -STR.y;
                ma = STR.x;
            }
            else
            {
                sc = STR.z;
                tc = -STR.t;
                ma = STR.x;
            }
        }
        else if(y > z)
        {
            if(STR.y > 0)
            {
                sc = STR.x;
                tc = STR.z;
                ma = STR.y;
            }
            else
            {
                sc = STR.x;
                tc = -STR.z;
                ma = STR.y;
            }
        }
        else
        {
            if(STR.z > 0)
            {
                sc = STR.x;
                tc = -STR.y;
                ma = STR.z;
            }
            else
            {
                sc = -STR.x;
                tc = -STR.y;
                ma = STR.z;
            }
        }
        return glm::vec2((sc/std::abs(ma) + 1.0) / 2.0,(tc/std::abs(ma) + 1.0) / 2.0);
    };

    glm::vec2 uv1 = get_uv(t1);
    glm::vec2 uv2 = get_uv(t2);
    glm::vec2 uv3 = get_uv(t3);

    glm::vec3 edge1 = p2 - p1;
    glm::vec3 edge2 = p3 - p1;

    glm::vec2 tedge1 = uv2 - uv1;
    glm::vec2 tedge2 = uv3 - uv1;

    float r = 1.0f / (tedge1.x * tedge2.y - tedge2.x - tedge1.y);

    glm::vec3 sdir((tedge2.y * edge1.x - tedge1.y * edge2.x) * r,
                   (tedge2.y * edge1.y - tedge1.y * edge2.y) * r,
                   (tedge2.y * edge1.z - tedge1.y * edge2.z) * r);

    glm::vec3 tdir((tedge1.x * edge2.x - tedge2.x * edge1.x) * r,
                   (tedge1.x * edge2.y - tedge2.x * edge1.y) * r,
                   (tedge1.x * edge2.z - tedge2.x * edge1.z) * r);

    m_vertices[i1].tangent[0] += sdir.x;
    m_vertices[i1].tangent[1] += sdir.y;
    m_vertices[i1].tangent[2] += sdir.z;

    m_vertices[i2].tangent[0] += sdir.x;
    m_vertices[i2].tangent[1] += sdir.y;
    m_vertices[i2].tangent[2] += sdir.z;

    m_vertices[i3].tangent[0] += sdir.x;
    m_vertices[i3].tangent[1] += sdir.y;
    m_vertices[i3].tangent[2] += sdir.z;

    storedTan[i1] += sdir;
    storedTan[i2] += sdir;
    storedTan[i3] += sdir;
}

for(int i = 0; i < m_vertices.size(); ++i)
{
    glm::vec3 n = glm::vec3(m_vertices[i].normal[0],m_vertices[i].normal[1],m_vertices[i].normal[2]);
    glm::vec3 t = glm::vec3(m_vertices[i].tangent[0],m_vertices[i].tangent[1],m_vertices[i].tangent[2]);

    glm::vec3 newT = glm::normalize(t - n * glm::dot(n,t));
    m_vertices[i].tangent[0] = newT.x;
    m_vertices[i].tangent[1] = newT.y;
    m_vertices[i].tangent[2] = newT.z;
    m_vertices[i].tangent[3] = (glm::dot(glm::cross(n,t), storedTan[i]) < 0.0f) ? -1.0f : 1.0f;
}

My VertexData looks like this BTW:

struct VertexData
{
    GLfloat position[4];
    GLfloat normal[3];
    GLfloat tcoords[3];
    GLfloat tangent[4];
};

I know that the current tcoords, position and normal are fine (otherwise you wouldn't see the screenshot above).

Then my vertex shader looks like this:

#version 400

layout (location = 0) in vec4 in_position;
layout (location = 1) in vec3 in_normal;
layout (location = 2) in vec3 in_UV;
layout (location = 3) in vec4 in_tangent;

struct PointLight
{
    bool active;

    vec3 position;
    vec3 ambient;
    vec3 diffuse;
    vec3 specular;

    float constant;
    float linear;
    float quadratic;
};

uniform mat4 model;
uniform mat4 view;
uniform mat4 projection;
uniform mat4 lightMVP;

uniform PointLight uLight;

smooth out vec3 ex_UV;
out vec3 ex_normal;
out vec3 ex_positionCameraSpace;
out vec3 ex_originalPosition;
out vec3 ex_positionWorldSpace;
out vec4 ex_positionLightSpace;
out vec3 ex_tangent;
out vec3 ex_binormal;

out PointLight ex_light;

void main()
{
    gl_Position = projection * view * model * in_position;

    ex_UV = in_UV;
    ex_normal = mat3(transpose(inverse(view * model))) * in_normal;
    ex_positionCameraSpace =  vec3(view * model * in_position);
    ex_originalPosition = vec3(in_position.xyz);
    ex_positionWorldSpace = vec3(model*in_position);
    ex_positionLightSpace = lightMVP * model * in_position;

    ex_tangent = mat3(transpose(inverse(view * model))) * in_tangent.xyz;
    ex_binormal = cross(ex_normal,ex_tangent);

    // provide the fragment shader with a light in view space rather than world space
    PointLight p = uLight;
    p.position = vec3(view * vec4(p.position,1.0));
    ex_light = p;
}

And finally my fragment shader looks like this:

#version 400

layout (location = 0) out vec4 color;

struct Material
{
    bool useMaps;
    samplerCube diffuse;
    samplerCube specular;
    samplerCube normal;
    float shininess;
    vec4 color1;
    vec4 color2;
};

struct PointLight
{
    bool active;

    vec3 position;
    vec3 ambient;
    vec3 diffuse;
    vec3 specular;

    float constant;
    float linear;
    float quadratic;
};

uniform Material uMaterial;

smooth in vec3 ex_UV;
in vec3 ex_normal;
in vec3 ex_positionCameraSpace;
in vec3 ex_originalPosition;
in vec3 ex_positionWorldSpace;
in vec4 ex_positionLightSpace;

in vec3 ex_tangent;
in vec3 ex_binormal;

in PointLight ex_light;

/* ******************
Provides a better lookup into a cubemap
******************* */
vec3 fix_cube_lookup(vec3 v, float cube_size)
{
    float M = max(max(abs(v.x), abs(v.y)), abs(v.z));
    float scale = (cube_size - 1) / cube_size;
    if (abs(v.x) != M)
        v.x *= scale;
    if (abs(v.y) != M)
        v.y *= scale;
    if (abs(v.z) != M)
        v.z *= scale;
    return v;
}

/* *********************
Calculates the color when using a point light. Uses shadow map
********************* */
vec3 CalcPointLight(PointLight light, Material mat, vec3 normal, vec3 fragPos, vec3 originalPos, vec3 viewDir)
{
    // replace the normal with lookup normal. This is now in tangent space
    vec3 textureLookup = fix_cube_lookup(normalize(ex_originalPosition),textureSize(mat.normal,0).x);
    normal = texture(mat.normal,textureLookup).rgb;

    // the direction the light is in in the light position - fragpos
    // light dir and view dir are now in tangent space
    vec3 lightDir = transpose(mat3(ex_tangent,ex_binormal,ex_normal)) * normalize(fragPos - light.position);
    viewDir = transpose(mat3(ex_tangent,ex_binormal,ex_normal)) * viewDir;

    // get the diffuse color
    textureLookup = fix_cube_lookup(normalize(ex_originalPosition),textureSize(mat.diffuse,0).x);
    vec3 diffuseMat = vec3(0.0);
    if(mat.useMaps)
        diffuseMat = texture(mat.diffuse,textureLookup).rgb;
    else
        diffuseMat = mat.color1.rgb;

    // get the specular color
    textureLookup = fix_cube_lookup(normalize(ex_originalPosition),textureSize(mat.specular,0).x);
    vec3 specularMat = vec3(0.0);
    if(mat.useMaps)
        specularMat = texture(mat.specular,textureLookup).rgb;
    else
        specularMat = mat.color2.rgb;

    // the ambient color is the amount of normal ambient light hitting the diffuse texture
    vec3 ambientColor = light.ambient * diffuseMat;

    // Diffuse shading
    float diffuseFactor = dot(normal, -lightDir);
    vec3 diffuseColor = vec3(0,0,0);
    vec3 specularColor = vec3(0,0,0);
    if(diffuseFactor > 0)
        diffuseColor = light.diffuse * diffuseFactor * diffuseMat;

    // Specular shading
    vec3 reflectDir = normalize(reflect(lightDir, normal));
    float specularFactor = pow(dot(viewDir,reflectDir), mat.shininess);
    if(specularFactor > 0 && diffuseFactor > 0)
        specularColor = light.specular * specularFactor * specularMat;

    float lightDistance = length(fragPos - light.position);
    float attenuation = light.constant + light.linear * lightDistance + light.quadratic * lightDistance * lightDistance;

    return ambientColor + (diffuseColor + specularColor) / attenuation;
}

void main(void)
{
    vec3 norm = normalize(ex_normal);
    vec3 viewDir = normalize(-ex_positionCameraSpace);

    vec3 result = CalcPointLight(ex_light,uMaterial,norm,ex_positionCameraSpace, ex_positionWorldSpace,viewDir);

    color = vec4(result,1.0);
}

As far as I can tell:

1. My tangents are being calculated correctly.
2. My normal map looks like a normal map to me.
3. I'm changing my light and view direction to tangent space to match my normal map.

The result is nothing. I.e. nothing is drawn to the screen. Not a solid color at all. So like everything behind is drawn with no occlusion.

If I discard the lookup into my normal map, and instead just use the tangent matrix light and view I get the following:

There's a post-processing lens flare on this that's producing those funny bits and bobs. What's important I think is the overwhelming glare from the surface where the normals seems somewhat accurate.

If I then just transform the light by the tangent matrix I get this:

All of this combines to tell me I have no idea where I'm going wrong.

I have an inkling that it's my tangent generation because the other pieces seem to follow what every tutorial I've read appear to say. The tangents are generated with a cubemapped icosphere in mind. So to determine the <S,T> or <U,V> 2D coordinates from a cubemaps usual 3D coordinates, I:

1. Use the largest value to determine the face I'm in
2. Use the code from https://www.opengl.org/registry/specs/ARB/texture_cube_map.txt to determine the S,T coordinates

Here's an excerpt from https://www.opengl.org/registry/specs/ARB/texture_cube_map.txt that I'm talking about.

  major axis
  direction     target                             sc     tc    ma
  ----------    -------------------------------    ---    ---   ---
   +rx          TEXTURE_CUBE_MAP_POSITIVE_X_ARB    -rz    -ry   rx
   -rx          TEXTURE_CUBE_MAP_NEGATIVE_X_ARB    +rz    -ry   rx
   +ry          TEXTURE_CUBE_MAP_POSITIVE_Y_ARB    +rx    +rz   ry
   -ry          TEXTURE_CUBE_MAP_NEGATIVE_Y_ARB    +rx    -rz   ry
   +rz          TEXTURE_CUBE_MAP_POSITIVE_Z_ARB    +rx    -ry   rz
   -rz          TEXTURE_CUBE_MAP_NEGATIVE_Z_ARB    -rx    -ry   rz

 Using the sc, tc, and ma determined by the major axis direction as
 specified in the table above, an updated (s,t) is calculated as
 follows

    s   =   ( sc/|ma| + 1 ) / 2
    t   =   ( tc/|ma| + 1 ) / 2

 This new (s,t) is used to find a texture value in the determined
 face's 2D texture image using the rules given in sections 3.8.5
 and 3.8.6." ...

EDIT I don't know why I didn't before, but I've output the normals, tangents and bitangents in a geometry shader to see the way they're facing. I used this tutorial.

Here they are

The yellows are the face normals, the greens are the vertex normals. I'm not sure why the vertex normals seem wrong, they don't affect any other lighting so it's probably just an error in my geometry shader.

Tangents are red, Binormals are blue. These seem (it's hard to tell) like they are prependicular to each-other, which is correct, but other than that they are not pointing in uniform directions. This is what's given the mottled sort of pattern I had before.

I have no idea how to fix this.

EDIT 2 I've figured out the problem with displaying my normals etc. This is fixed now.

The result, I added some shading to make it clearer, each color is a different cube face.

Something else I've changed is the lookup into my normal map. I forgot to adjust the range back into -1 to 1 (from 0 to 1).

normal = texture(mat.normal,textureLookup).rgb * 2.0 - 1.0;

This doesn't fix my problem.

The confusing part is that when I try using the normals from my texture, I don't get anything rendered. Nothing at all into the depth buffer. I've checked and double checked that the texture is accessible from the shader (hence the original screenshot showing the texture applied to the sphere).

Because even though my Tangents and Binormals are pointing every which way; I'd still expect something to be shown, even if it's wrong. But not even the ambient color is coming through. (this happens even if I leave my lightDir and viewdir alone. If I just ignore the vertex normal and lookup the texture. I lose ambient color)...

EDIT 3: One last problem

As is often the case, part of the problem had nothing to do with where you think it's wrong. My problem was that I was overwriting the binding of my normal map with a different texture.

So, with that out of the way, I can now see my colours come through. With my nice sexy bump mapping.

However, there's now a problem at the seams of the cubemap. I'm not sure if it's because of the tangents being calculated or because of the way my normal map is generated. My normal map is generated from a height map for each face, independently.

This would explain some seam affect I think, I'm going to modify it to sample the adjacent face on those edges and see what happens.

I still think that the tangents being generated are also going to have an adverse affect on these seams as well. My thoughts is that they are going to be pointing in opposite directions at the seams.

Screenshot:

EDIT 4 While testing from EDIT1 down I was using a very very low poly mesh for my icosphere. So I had minimal sub-divisions.

I wanted to see how my not quite perfect normal mapped sphere looked with lots of polys. This instantly revealed this problem:

In case it's not clear, running from left to write is my old friend, the seam, but below that are, what appears to be, triangle edges.

So after all of the above, I think I'm back to my original problem of incorrect tangents.

Still looking for some help from anyone that's reading this.

EDIT 4 Well, that was quick. This site here http://www.geeks3d.com/20130122/normal-mapping-without-precomputed-tangent-space-vectors/ gave me another way to create tangents. Whilst the code seems somewhat similar to what I was doing on the CPU, it's not resulting in those randomly oriented tangents that was producing those edges from EDIT 3.

I am very close now. I still have the seams, this other method of generating the tangents seems to have increased their "seaminess"

EDIT 5 I've now tried modifying my normal map generation. The previous code went like this:

for(int i = 0; i < 6; ++i)
{   
    float scale = 15.0;
    std::deque<glm::vec4> normalMap(textureSize*textureSize);
    for(int x = 0; x < textureSize; ++x)
    {
        for(int y = 0; y < textureSize; ++y)
        {
            // center point
            int i11 = utils::math::get_1d_array_index_from_2d(x,y,textureSize);
            float v11 = cubeFacesHeight[i][i11].r;

            // to the left
            int i01 = utils::math::get_1d_array_index_from_2d(std::max(x-1,0),y,textureSize);
            float v01 = cubeFacesHeight[i][i01].r;

            // to the right
            int i21 = utils::math::get_1d_array_index_from_2d(std::min(x+1,textureSize-1),y,textureSize);
            float v21 = cubeFacesHeight[i][i21].r;

            // to the top
            int i10 = utils::math::get_1d_array_index_from_2d(x,std::max(y-1,0),textureSize);
            float v10 = cubeFacesHeight[i][i10].r;

            // and now the bottom
            int i12 = utils::math::get_1d_array_index_from_2d(x,std::min(y+1,textureSize-1),textureSize);
            float v12 = cubeFacesHeight[i][i12].r;

            glm::vec3 S = glm::vec3(1, 0, scale * v21 - scale * v01);
            glm::vec3 T = glm::vec3(0, 1, scale * v12 - scale * v10);

            glm::vec3 N = (glm::vec3(-S.z,-T.z,1) / std::sqrt(S.z*S.z + T.z*T.z + 1));

            N.x = (N.x+1.0)/2.0;
            N.y = (N.y+1.0)/2.0;
            N.z = (N.z+1.0)/2.0;
            normalMap[utils::math::get_1d_array_index_from_2d(x,y,textureSize)] = glm::vec4(N.x,N.y,N.z,v11);
        }
    }
    for(int x = 0; x < textureSize; ++x)
    {
        for(int y = 0; y < textureSize; ++y)
        {
            cubeFacesHeight[i][utils::math::get_1d_array_index_from_2d(x,y,textureSize)] = normalMap[utils::math::get_1d_array_index_from_2d(x,y,textureSize)];
        }
    }
}

cubeFacesHeight is an std::array of 6 std::deques of glm::vec4s. Or, the six sides of my cubemap. The colors in the faces are greyscale, I'm not using floats for reasons that don't matter.

I've now changed it to the following, warning, this is ugly and long.

for(int i = 0; i < 6; ++i)
{
    // 0 is negative X
    // 1 is positive X
    // 2 is negative Y
    // 3 is positive Y
    // 4 is negative Z
    // 5 is positive Z

    // +X:  right -Z (left),    left +Z (right),    top -Y (right),     bottom +Y (right)
    // -X:  right +Z (left),    left -Z (right),    top -Y (left),      bottom +Y (left)
    // -Z:  right -X (left),    left +X (right),    top -Y (bottom),    bottom +Y (top)
    // +Z:  right +X (left),    left -X (right),    top -Y (top),       bottom +Y (bottom)
    // -Y:  right +X (top),     left -X (top),      top +Z (top),       bottom -Z (top)
    // +Y:  right +X (bottom),  left -X (bottom),   top -Z (bottom),    bottom +Z (bottom)

    //+Z is towards, -Z is distance
    const int NEGATIVE_X = 0;
    const int NEGATIVE_Y = 2;
    const int NEGATIVE_Z = 4;
    const int POSITIVE_X = 1;
    const int POSITIVE_Y = 3;
    const int POSITIVE_Z = 5;

    float scale = 15.0;
    std::deque<glm::vec4> normalMap(textureSize*textureSize);
    for(int x = 0; x < textureSize; ++x)
    {
        for(int y = 0; y < textureSize; ++y)
        {
            // center point
            int i11 = utils::math::get_1d_array_index_from_2d(x,y,textureSize);
            float v11 = cubeFacesHeight[i][i11].r;

            // to the left
            int i01 = utils::math::get_1d_array_index_from_2d(std::max(x-1,0),y,textureSize);
            float v01 = cubeFacesHeight[i][i01].r;
            if(x-1 < 0)
            {
                if(i == NEGATIVE_X)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(textureSize-1,y,textureSize);
                    v01 = cubeFacesHeight[NEGATIVE_Z][i01].r;
                }
                else if(i == POSITIVE_X)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(textureSize-1,y,textureSize);
                    v01 = cubeFacesHeight[POSITIVE_Z][i01].r;
                }
                else if(i == NEGATIVE_Z)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(textureSize-1,y,textureSize);
                    v01 = cubeFacesHeight[POSITIVE_X][i01].r;
                }
                else if(i == POSITIVE_Z)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(textureSize-1,y,textureSize);
                    v01 = cubeFacesHeight[NEGATIVE_X][i01].r;
                }
                else if(i == NEGATIVE_Y)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(y,0,textureSize);
                    v01 = cubeFacesHeight[NEGATIVE_X][i01].r;
                }
                else if(i == POSITIVE_Y)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(y,textureSize-1,textureSize);
                    v01 = cubeFacesHeight[NEGATIVE_X][i01].r;
                }
            }

            // to the right
            int i21 = utils::math::get_1d_array_index_from_2d(std::min(x+1,textureSize-1),y,textureSize);
            float v21 = cubeFacesHeight[i][i21].r;
            if(x+1 > textureSize-1)
            {
                if(i == NEGATIVE_X)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(0,y,textureSize);
                    v01 = cubeFacesHeight[POSITIVE_Z][i01].r;
                }
                else if(i == POSITIVE_X)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(0,y,textureSize);
                    v01 = cubeFacesHeight[NEGATIVE_Z][i01].r;
                }
                else if(i == NEGATIVE_Z)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(0,y,textureSize);
                    v01 = cubeFacesHeight[NEGATIVE_X][i01].r;
                }
                else if(i == POSITIVE_Z)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(0,y,textureSize);
                    v01 = cubeFacesHeight[POSITIVE_X][i01].r;
                }
                else if(i == NEGATIVE_Y)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(y,0,textureSize);
                    v01 = cubeFacesHeight[POSITIVE_X][i01].r;
                }
                else if(i == POSITIVE_Y)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(y,textureSize-1,textureSize);
                    v01 = cubeFacesHeight[POSITIVE_X][i01].r;
                }
            }

            // to the top
            int i10 = utils::math::get_1d_array_index_from_2d(x,std::max(y-1,0),textureSize);
            float v10 = cubeFacesHeight[i][i10].r;
            if(y-1 < 0)
            {
                if(i == NEGATIVE_X)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(0,x,textureSize);
                    v01 = cubeFacesHeight[NEGATIVE_Y][i01].r;
                }
                else if(i == POSITIVE_X)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(textureSize-1,x,textureSize);
                    v01 = cubeFacesHeight[NEGATIVE_Y][i01].r;
                }
                else if(i == NEGATIVE_Z)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(x,textureSize-1,textureSize);
                    v01 = cubeFacesHeight[NEGATIVE_Y][i01].r;
                }
                else if(i == POSITIVE_Z)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(x,0,textureSize);
                    v01 = cubeFacesHeight[NEGATIVE_Y][i01].r;
                }
                else if(i == NEGATIVE_Y)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(x,0,textureSize);
                    v01 = cubeFacesHeight[POSITIVE_Z][i01].r;
                }
                else if(i == POSITIVE_Y)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(x,textureSize-1,textureSize);
                    v01 = cubeFacesHeight[NEGATIVE_Z][i01].r;
                }
            }

            // and now the bottom
            int i12 = utils::math::get_1d_array_index_from_2d(x,std::min(y+1,textureSize-1),textureSize);
            float v12 = cubeFacesHeight[i][i12].r;
            if(y+1 > textureSize-1)
            {
                if(i == NEGATIVE_X)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(0,x,textureSize);
                    v01 = cubeFacesHeight[POSITIVE_Y][i01].r;
                }
                else if(i == POSITIVE_X)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(textureSize-1,x,textureSize);
                    v01 = cubeFacesHeight[POSITIVE_Y][i01].r;
                }
                else if(i == NEGATIVE_Z)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(x,0,textureSize);
                    v01 = cubeFacesHeight[POSITIVE_Y][i01].r;
                }
                else if(i == POSITIVE_Z)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(x,textureSize-1,textureSize);
                    v01 = cubeFacesHeight[POSITIVE_Y][i01].r;
                }
                else if(i == NEGATIVE_Y)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(x,0,textureSize);
                    v01 = cubeFacesHeight[NEGATIVE_Z][i01].r;
                }
                else if(i == POSITIVE_Y)
                {
                    i01 = utils::math::get_1d_array_index_from_2d(x,textureSize-1,textureSize);
                    v01 = cubeFacesHeight[POSITIVE_Z][i01].r;
                }
            }

            glm::vec3 S = glm::vec3(1, 0, scale * v21 - scale * v01);
            glm::vec3 T = glm::vec3(0, 1, scale * v12 - scale * v10);

            glm::vec3 N = (glm::vec3(-S.z,-T.z,1) / std::sqrt(S.z*S.z + T.z*T.z + 1));

            N.x = (N.x+1.0)/2.0;
            N.y = (N.y+1.0)/2.0;
            N.z = (N.z+1.0)/2.0;

            normalMap[utils::math::get_1d_array_index_from_2d(x,y,textureSize)] = glm::vec4(N.x,N.y,N.z,v11);
        }
    }
    for(int x = 0; x < textureSize; ++x)
    {
        for(int y = 0; y < textureSize; ++y)
        {
            cubeFacesHeight[i][utils::math::get_1d_array_index_from_2d(x,y,textureSize)] = normalMap[utils::math::get_1d_array_index_from_2d(x,y,textureSize)];
        }
    }
}

So I'm now sort of "bleeding" into the adjacent cubeface to sample the height there while generating the normalmap. This has actually increased the seam appearance.

But this sort of raises it's own questions. For instance... "why in the hell is the affect increased?" You can see that it's now a sort of bevel effect.

So, I'm fairly sure I've matched up my cubefaces correctly when "bleeding" into the next one. This brings me back to the tangents being incorrect.

Even if I completely mixed up the cube faces, it wouldn't give a bevel effect, it would be something completely spotty. For example, even on a completely flat section, i.e., bleeding the normal map generation into the next face would have zero effect, I still see a massive bevel.

This makes me think that if the tangents were correct before, the normal map sort of "matched" the tangent directions? I don't know.

quick edit I noticed that I was effectively sampling my face edges twice during my original map generation. If I remove this double sampling and just use 0 for the additional, I end up seeing those same big seams. I'm not sure what this means...

Another quick edit This image shows something that I think is very telling.

I can see here that two different faces are "pointing" in opposite directions. This is with my in fragment tangent generation.

So I'm back to my tangents being a problem.

Historicity

@kfsone thanks for the advice, but it didnt need your pissy comment at the start. I think the upvotes tell me there's nothing wrong with my question. Ill ask on the gamedev site though (that is good advice). The mods can close my question if this is a good example of how not to ask questions. – Historicity

If there is a way you could send me your project in full, all source code with all assets, I would not have a problem looking at the source and checking it out. It is a tough question to answer, for it isn't just the calculations of the tangents, normals, bitagents & binormals that are involved it is also the construction of your sphere and how the vertices are laid out along with how you are defining your texture coordinates. (...) – Millesimal

(continued...) When I first started to work with 3D renderings and was generating basic 3D shapes such as a Cylinder and assigning texture coordinates to be seamless and working properly with lighting, when defining the normals I had to take into consideration that each vertex that is co-defined between multiple triangle faces was generating multiple normals. To correct the issue I had to take an average from all normals that were generated for a specific vertex. I will display an illustration in the answer section although it is not an answer or solution to your problem. – Millesimal

maybe you want to use gamedev.net graphics forum for this kind of thread. – Barsac

Answer 1

Normal Maps work best with the normal vector matrices that originally created the normal map

I believe your problem has to do with non uniform alignment of your tangents across the surface. UV mapping is generally the first place to look with issues like this. And, mapping a sphere with a 2D image ain't so easy (look at all the various earth projection topologies and you'll see what I mean). At some point, you're going to get stretching, edges, or shearing, and most likely some combination of all of the above. Usually with UV mapping, the point is to choose where you're going to hide these effects on the surface. The poles of planets are often chosen for this. One place I would look would be to realign your tangents and binormals so that they're all sharing a common, global orientation, ie. tanget = north, and binormal = east, with the normal facing out (altitude). The non-uniformity of your tangents and binormals plays a direct role in the artifacts that sometimes arise in normal mapping issues, because they can twist the effect of the normal map, at that location, if the normal map was baked with the assumption that all tangents and binormals are uniformly oriented.

In essence, the normal map was baked/created with an implicit understanding of your tangents and binormals. If, when the normal map is reapplied, the tangents and binormals of the surface do not align with the implicit understanding in which the normal map was originally created, then you will get lighting and shading errors.

The benefit or orthogonal normal vector matrix

The benefit of this is that the tangent and binormal vectors are often used to lookup the 2D texture coordinate. If your matrix is non-orthogonal, then you run the risk ofshearing, rotations, or loss of precision at skewed angles.

---

Define Uniform, Orthogonal Normal Vector Matrices

You could approach your normal/tangent/binormal calculations in a different waut that would assure two factors:

1. uniform object orientation, ie... all pointing in the same relative direction
2. orthogonal vectors, which will limit texture lookup shearing

This will work by transforming a predefined, orthogonal vector matrix through two rotations and one move. For the sake of explanation, I won't collapse those three matrix operations into a single matrix, but it might behoove you to do so in your code.

First, start with an already defined vector matrix

vec3 = [1, 0, 0, 0, 1, 0, 0, 0, 1];

Second, perform these operations in object space, not world space

Otherwise you'll have to transform that object back to world center, and rotate it back to it's origin orientation, then apply the normal transformations, then send the object back to it's work position and orientation

Third, create a vector from vtx[n] to the object center

This vector will tell you how much to rotate your normal vector matrix in two directions:

• Longitudinal rotation
• Latitudinal rotation

Fourth, Rotate your normal vector matrix to align

Last, Move your normal vector matrix by the distance

Rinse and Repeat

---

If you needed to keep non-uniform, non-orthogonal UVs

You can create a normal map based on incongruous UV layout such that it would take that layout into effect and therefore appropriately apply itself without effect. But your normal map would have to be created from this innate incongruity so that it would elegantly apply itself to those UVs.

---

Edge Pixel Interpolation?

Third, looking at how the edge of the normal map crease follows the shape of the cubemap, I was wondering how you're interpolating edge pixels for your normal map.

---

GLSL texture lookup for cubemaps?

Also, and I maybe I just didn't find the section of your answer where you address this, but have you considered using the GLSL cubemap lookup function? gvec4 texture( gsamplerCube sampler, vec3 P, [float bias]);

Hoeg

Answer 2

Here I is the illustration that I mentioned in the comment:

As you can see the red lines are a generated normal and each vertex at the base has two. This causes lighting problems because the face of each triangle is in a different direction. When I first came across this I had to take an average of both normals on each vertex represented by the yellow lines to fix lighting calculations.

As for the seem you are getting from your cube - bump map this could be in how you are generating your vertices to define your sphere and how you are applying your texture coordinates. I can not tell directly without seeing your whole solution or project and working with it. The problem may not even involve your tangents but could be within your texture mapping causing a wrapping effect.

This is not a direct answer to your problem but suggestions to be aware of since there are many different ways to implement these types of shaders and renderings.

Millesimal

I'm generating the sphere as an Icosphere. So the vertices themselves are not aligned on the faces of a cube. The normals are simply direction vectors from the center of the sphere. So there's no need to generate per-face vs per-vertex normals or averaging them or anything like that. They are very straight forward. There's nothing wrong with my texture coords as any other cubemap texture mapping works perfectly fine. I imagine the seams are from the way I'm generating the normals per face. – Historicity

I've actually got a somewhat satisfactory result today. thelastboundary.com/wp-content/uploads/2015/06/… taking the idea from rorydriscoll.com/2012/01/11/derivative-maps I'm still trying to decipher what it means in the "What Now?" section as that seems to give a much smoother result than the pixelated affect I'm getting now – Historicity

Okay that is good to hear that you are progressing. As I have said though without having the project to run on my end and working with some numbers and equations, I can not tell where the problem is. Just from reading it I do not see anything out of the ordinary that sticks out. I don't know if it makes a difference or not, but I noticed you are using GLM which is good, but which version are you building against? – Millesimal

Since partial derivatives where being used, they were talking about using them to transfer from one coordinate space to another within the algorithm by applying the chain rule. Doing this allows the shader to interpolate between derivatives when the object's position is closer to the view camera. Without using the chain rule to achieve this, then as in the first example when the object gets closer to the camera you can see chunks of blocks instead of a smoother rendering. – Millesimal

Yes, so I sort of undertstand this. What I don't understand is "texel-space" and the screen-space UV derivatives. So I take it that I should no longer use my float height values but instead I should have a vec2; but how do I compute this. Second, is the screenspace uv derivative, I don't understand this at all. – Historicity

It is a bit complicated and complex and tough to grasp at first, it is also not easy to explain. What I can gather from reading that page, it is a different method as opposed to using tangents, where by using a gradient that has a vector field, it is the derivatives of that you are using as your map. – Millesimal

Answer 3

It took me a long time back in the day to understand how to compute tangent space. Maybe the way I finally got it can help.

You have three vertices v0, v1, v2. Each has a position, normal and uv. Let's compute the tangent space for v0. The z axis will be v0.normal. We need to compute the x and y axis.

Any point on the triangle can be expressed as v0.pos + (v1.pos-v0.pos)*t + (v2.pos-v0.pos)*s. Any texture coordinate can be expressed as v0.uv + (v1.uv - v0.uv)*t + (v2.uv - v0.uv)*s.

In tangent space we need to have v1.uv - v0.uv = (1,0) and v2.uv-v0.uv = (0,1). We can solve for that s,t! for both cases! And that is the s and t for our tangent and binormal. Just plug them back into the position equation and you have the position where uv=(0,1) and uv=(1,0). Subtract v0.pos and you have your x and y axis! Also normalize them.

And that is you tangent space for v0. A 3x3 matrix. It's not orthogonal necessarily. But that is ok. Also you compute this matrix per vertex for every triangle using that vertex. Just average them.

Interpolate those per vertex matrices when rendering, and normalize them per pixel.

A good way to test is to just render the z column - it should be the normal.

For lighting subtract the interpolated position from the light and transform it by the "tangent matrix". Now your light is in tangent space, where (0,0,1) is towards the light, and normal maps point straight up.

Poach

If the original normal map was created with orthogonal tangents, normals, and binormals, then you will want to ensure the the 3D surface does have orthogonal tangents, normals, and binormals. Often times, the tangent and binormal vectors are used as the u & v lookup vectors. Ie... tangent = v, and binormal = u (or vice versa). Within the normal map, only two channels (R&G) relate to the directionality of the surface lighting, with the blue channel relating to how much the surface normal is warped by the R&G channels. – Hoeg