OpenGL Shadow Map

I am trying to do a basic shadow map but for some reason, It doesn't render properly.

Video of the Problem

I render the house using a flat shader:

int shadowMapWidth = WINDOW_SIZE_X * (int)SHADOW_MAP_RATIO;
int shadowMapHeight =  WINDOW_SIZE_Y * (int)SHADOW_MAP_RATIO;

// Rendering into the shadow texture.
glActiveTexture(GL_TEXTURE0);
CALL_GL(glBindTexture(GL_TEXTURE_2D, shadowTexture));
// Bind the framebuffer.
CALL_GL(glBindFramebuffer(GL_FRAMEBUFFER, shadowFBO));
//Clear it
CALL_GL(glClear(GL_DEPTH_BUFFER_BIT));
CALL_GL(glViewport(0, 0, shadowMapWidth, shadowMapHeight));
CALL_GL(glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE));
//Render stuff
flatShader.use();
flatShader["baseColor"] = glm::vec4(1.0f,1.0f,1.0f,1.0f);
flatShader["pvm"] = projectionMatrix*pointLight.viewMatrix*cursor.modelMatrix;
cursor.draw(); //binds the vao and draws

// Revert for the scene.
CALL_GL(glBindFramebuffer(GL_FRAMEBUFFER, 0));
CALL_GL(glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE));
CALL_GL(glViewport(0, 0, WINDOW_SIZE_X, WINDOW_SIZE_Y));

Notice that I only render the house. I don't render the floor in the depth-buffer pass.

Following this I render the quad that represents the floor using the following shader pair:

/* [VERT] */ 
#version 330

in vec3 in_Position;
in vec2 in_TexCoord;

uniform mat4 shadowMatrix;
uniform mat4 mvp;

out vec2 UV;
out vec4 shadowProj;

void main()
{
    gl_Position = mvp*vec4(in_Position,1.0);
    shadowProj = shadowMatrix*vec4(in_Position,1.0);
    UV = in_TexCoord;
}

And the Fragment Shader:

/* [FRAG] */ 
#version 330

in vec2 UV;
in vec4 shadowProj;

out vec4 fragColor;

uniform sampler2D texturex;
uniform sampler2DShadow shadowMap;

void main()
{
    fragColor = vec4(texture(texturex, UV).rgb,1);

    float shadow = 1.0;
    shadow = textureProj(shadowMap,shadowProj);

    fragColor *= shadow;

}

I then draw the house again in color and ... the floor:

textureShader.use();

glUniform1i(baseImageLoc, 0); //Texture unit 0 is for base images.
glUniform1i(shadowMapLoc, 1); //Texture unit 1 is for shadow maps.

glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, floorTexture);

glActiveTexture(GL_TEXTURE1);
glBindTexture(GL_TEXTURE_2D, shadowTexture);
textureShader["shadowMatrix"] = projectionMatrix*pointLight.viewMatrix*floorMatrix;
textureShader["mvp"] = projectionMatrix*viewMatrix*floorMatrix;
CALL_GL(glBindVertexArray(floorVAO));
CALL_GL(glDrawArrays(GL_TRIANGLES,0,18));

glfwSwapBuffers();

Has anybody seen this behavior before? Any idea what could be wrong? By the way, the light's coordinates place it directly on top of the house so the shadow should be directly below the house on the floor (but it ends up sideways).

For reference here is how I generate the shadow FBO:

int shadowMapWidth = WINDOW_SIZE_X * (int)SHADOW_MAP_RATIO;
int shadowMapHeight =  WINDOW_SIZE_Y * (int)SHADOW_MAP_RATIO;

glGenTextures(1, &shadowTexture);
glBindTexture(GL_TEXTURE_2D, shadowTexture);
glTexImage2D(GL_TEXTURE_2D, 0, GL_DEPTH_COMPONENT, shadowMapWidth, shadowMapHeight, 0, GL_DEPTH_COMPONENT, GL_UNSIGNED_BYTE, 0);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_MODE, GL_COMPARE_REF_TO_TEXTURE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_COMPARE_FUNC, GL_LESS);
glTexImage2D(GL_TEXTURE_2D,0,GL_DEPTH_COMPONENT,shadowMapWidth,shadowMapHeight,0,GL_DEPTH_COMPONENT,GL_FLOAT,NULL);
glTexParameteri(GL_TEXTURE_2D,GL_TEXTURE_COMPARE_MODE,GL_COMPARE_R_TO_TEXTURE);
glBindTexture(GL_TEXTURE_2D, 0); //unbind the texture

glGenFramebuffers(1, &shadowFBO);
glBindFramebuffer(GL_FRAMEBUFFER, shadowFBO);
glFramebufferTexture2D(GL_FRAMEBUFFER, GL_DEPTH_ATTACHMENT, GL_TEXTURE_2D, shadowTexture, 0);
glDrawBuffer(GL_NONE);
glReadBuffer(GL_NONE);

if (glCheckFramebufferStatus(GL_FRAMEBUFFER) != GL_FRAMEBUFFER_COMPLETE)
{ printf("GL_FRAMEBUFFER_COMPLETE error 0x%x", glCheckFramebufferStatus(GL_FRAMEBUFFER)); }

glClearDepth(1.0f); glEnable(GL_DEPTH_TEST);
// Needed when rendering the shadow map. This will avoid artifacts.
glPolygonOffset(1.0f, 0.0f); glBindFramebuffer(GL_FRAMEBUFFER, 0);
//to convert the texture coordinates to -1 ~ 1
   GLfloat biasMatrixf[] = {
    0.5f, 0.0f, 0.0f, 0.0f,
    0.0f, 0.5f, 0.0f, 0.0f,
    0.0f, 0.0f, 0.5f, 0.0f,
    0.5f, 0.5f, 0.5f, 1.0f };

biasMatrix = glm::make_mat4(biasMatrixf);

It looks like you forgot to multiply your shadow matrix by the bias matrix.