This causes a stack smash when compiled with GCC.
void MyOpenGLLineClass::set_width (float x)
{
float max = 1;
glGetFloatv (GL_ALIASED_LINE_WIDTH_RANGE, &max);
if (false == (x > 0 && x <= max))
throw std::invalid_argument (__PRETTY_FUNCTION__);
m_width = x;
}
It happens in a program which otherwise behaves as expected. If I comment-out the glGetFloatv
the stack smash goes away.
Here is the machine code:
0x00005555556503be <+0>: push %rbp
0x00005555556503bf <+1>: mov %rsp,%rbp
0x00005555556503c2 <+4>: push %r12
0x00005555556503c4 <+6>: push %rbx
0x00005555556503c5 <+7>: sub $0x20,%rsp
0x00005555556503c9 <+11>: callq *0x2e2bf9(%rip) # 0x555555932fc8
0x00005555556503cf <+17>: mov %rdi,-0x28(%rbp)
0x00005555556503d3 <+21>: movss %xmm0,-0x2c(%rbp)
0x00005555556503d8 <+26>: mov %fs:0x28,%rax
0x00005555556503e1 <+35>: mov %rax,-0x18(%rbp)
0x00005555556503e5 <+39>: xor %eax,%eax
0x00005555556503e7 <+41>: pxor %xmm0,%xmm0
0x00005555556503eb <+45>: movss %xmm0,-0x1c(%rbp)
0x00005555556503f0 <+50>: lea -0x1c(%rbp),%rax
0x00005555556503f4 <+54>: mov %rax,%rsi
0x00005555556503f7 <+57>: mov $0x846e,%edi
0x00005555556503fc <+62>: callq 0x5555556262d0 <glGetFloatv@plt>
0x0000555555650401 <+67>: movss -0x2c(%rbp),%xmm0
0x0000555555650406 <+72>: pxor %xmm1,%xmm1
0x000055555565040a <+76>: comiss %xmm1,%xmm0
0x000055555565040d <+79>: seta %al
0x0000555555650410 <+82>: xor $0x1,%eax
0x0000555555650413 <+85>: test %al,%al
0x0000555555650415 <+87>: jne 0x55555565042a <MyOpenGLLineClass::set_width(float)+108>
0x0000555555650417 <+89>: movss -0x1c(%rbp),%xmm0
0x000055555565041c <+94>: comiss -0x2c(%rbp),%xmm0
0x0000555555650420 <+98>: setae %al
0x0000555555650423 <+101>: xor $0x1,%eax
0x0000555555650426 <+104>: test %al,%al
0x0000555555650428 <+106>: je 0x55555565045f <MyOpenGLLineClass::set_width(float)+161>
0x000055555565042a <+108>: mov $0x10,%edi
0x000055555565042f <+113>: callq 0x555555626130 <__cxa_allocate_exception@plt>
0x0000555555650434 <+118>: mov %rax,%rbx
0x0000555555650437 <+121>: lea 0x8ee42(%rip),%rsi # 0x5555556df280 <_ZZN3CMyOpenGLLineClass9set_widthEfE19__PRETTY_FUNCTION__>
0x000055555565043e <+128>: mov %rbx,%rdi
0x0000555555650441 <+131>: callq 0x555555625f80 <_ZNSt16invalid_argumentC1EPKc@plt>
0x0000555555650446 <+136>: mov 0x2e2b5b(%rip),%rax # 0x555555932fa8
0x000055555565044d <+143>: mov %rax,%rdx
0x0000555555650450 <+146>: lea 0x2de9f1(%rip),%rsi # 0x55555592ee48 <_ZTISt16invalid_argument@@GLIBCXX_3.4>
0x0000555555650457 <+153>: mov %rbx,%rdi
0x000055555565045a <+156>: callq 0x555555625a80 <__cxa_throw@plt>
0x000055555565045f <+161>: mov -0x28(%rbp),%rax
0x0000555555650463 <+165>: movss -0x2c(%rbp),%xmm0
0x0000555555650468 <+170>: movss %xmm0,0x58(%rax)
0x000055555565046d <+175>: nop
0x000055555565046e <+176>: mov -0x18(%rbp),%rax
0x0000555555650472 <+180>: xor %fs:0x28,%rax
0x000055555565047b <+189>: je 0x55555565049a <MyOpenGLLineClass::set_width(float)+220>
0x000055555565047d <+191>: jmp 0x555555650495 <MyOpenGLLineClass::set_width(float)+215>
0x000055555565047f <+193>: mov %rax,%r12
0x0000555555650482 <+196>: mov %rbx,%rdi
0x0000555555650485 <+199>: callq 0x5555556263c0 <__cxa_free_exception@plt>
0x000055555565048a <+204>: mov %r12,%rax
0x000055555565048d <+207>: mov %rax,%rdi
0x0000555555650490 <+210>: callq 0x555555625b00 <_Unwind_Resume@plt>
0x0000555555650495 <+215>: callq 0x5555556258a0 <__stack_chk_fail@plt>
=> 0x000055555565049a <+220>: add $0x20,%rsp
0x000055555565049e <+224>: pop %rbx
0x000055555565049f <+225>: pop %r12
0x00005555556504a1 <+227>: pop %rbp
0x00005555556504a2 <+228>: retq
Why is this stack smash happening?
According to the Khronos reference, glGetFloatv
with GL_ALIASED_LINE_WIDTH_RANGE
returns two values:
GL_ALIASED_LINE_WIDTH_RANGE
params returns two values, the smallest and largest supported widths for aliased lines.
So the second value invokes UB and smashes your stack.
Fixed:
void MyOpenGLLineClass::set_width (float x)
{
float range[] = {0,0};
glGetFloatv (GL_ALIASED_LINE_WIDTH_RANGE, range);
if (false == (x >= range[0] && x <= range[1]))
throw std::invalid_argument (__PRETTY_FUNCTION__);
m_width = x;
}