I'm trying to build a simple cooperative multithreading library in C. Basically, it is possible to create threads using thread_create, add them to the runqueue with thread_queue, and then execute them to completion using thread_exec. Within the function that is assigned to a thread, it is possible to call thread_yield to put the current thread at the end of the runqueue and continue with the next thread.
Because the threads may be interrupted, their stacks are heap-allocated. I then use setjmp in thread_yield to remember the current state of execution, and longjmp in the dispatch-function to resume a thread.
For some reason, when running with compiler optimizations, I get the following error:
*** longjmp causes uninitialized stack frame ***: terminated
I do not get this error when compiling with -D_FORTIFY_SOURCE=0, and get the expected output:
started at main
printing from 1, now yielding
printing from 2, now yielding
printing from 3, now yielding
printing from 1, now yielding
printing from 2, now yielding
printing from 3, now yielding
printing from 1, now yielding
printing from 2, now yielding
printing from 3, now yielding
returned from thread
returned from thread
returned from thread
finished at main
I tried implementing saving and restoring of the rsp-register, but to no avail. I'd like to get this working without disabling compiler inserted safety checks and would appreciate any suggestions.
Here's the full code:
threads.c
#include "threads.h"
#include <stdlib.h>
#include <stdio.h>
struct thread* head_thread = NULL;
struct thread* tail_thread = NULL;
size_t initial_rbp = 0;
jmp_buf threading_start_ctx;
// create a thread executing f(arg)
struct thread* thread_create(void (*f)(void*), void* arg) {
// allocate stack of one megabyte
size_t stack_size = 1 << 20;
struct thread* thread = (struct thread*) malloc(sizeof(struct thread));
thread->next = NULL;
thread->f = f;
thread->arg = arg;
thread->stack_ptr = malloc(stack_size);
thread->stack_size = stack_size;
thread->has_run = false;
thread->rbp = 0;
return thread;
}
// add a thread to back queue
void thread_queue(struct thread* thread) {
if (!tail_thread) {
head_thread = thread;
} else {
tail_thread->next = thread;
}
tail_thread = thread;
thread->next = NULL;
}
// yield from thread, transfering execution to another thread
void thread_yield(void) {
// save rbp
asm("movq %%rbp, %[RBP]" : [RBP] "=rm" (head_thread->rbp) :);
if (!setjmp(head_thread->jmp_buf)) {
// if jmp was just set, schedule a different thread and execute it
schedule();
dispatch();
}
// restore rbp
asm("movq %[RBP], %%rbp" : : [RBP] "rm" (head_thread->rbp));
}
// reschedule threads
static void schedule(void) {
// only swap if more than one thread
if (head_thread != tail_thread) {
struct thread* current = head_thread;
head_thread = head_thread->next;
tail_thread->next = current;
tail_thread = current;
tail_thread->next = NULL;
}
}
// execute current head thread
static void dispatch(void) {
if (head_thread) {
// stack grows downwards, therefore add offset
size_t stack_top = (size_t) head_thread->stack_ptr + head_thread->stack_size;
if (!head_thread->has_run) {
head_thread->has_run = true;
// set up rbp and rsp to thread stack
asm(
"movq %[StackTop], %%rbp\n"
"movq %[StackTop], %%rsp"
: : [StackTop] "r" (stack_top)
);
// execute thread
head_thread->f(head_thread->arg);
printf("returned from thread\n");
// returned from thread, load starting rbp
asm("movq %[RBP], %%rbp" : : [RBP] "rm" (initial_rbp));
longjmp(threading_start_ctx, 1);
} else {
// load rbp of current head and continue where it yielded
asm("movq %[RBP], %%rbp": : [RBP] "rm" (head_thread->rbp));
longjmp(head_thread->jmp_buf, 1);
}
}
}
// start executing queued threads
void thread_exec(void) {
// save starting rbp
asm("movq %%rbp, %[RBP]" : [RBP] "=rm" (initial_rbp) :);
if (setjmp(threading_start_ctx)) {
// if arrive from longjmp, free current head
struct thread* next = head_thread->next;
free(head_thread->stack_ptr);
free(head_thread);
head_thread = next;
if (!head_thread) {
tail_thread = NULL;
}
}
if (head_thread) {
dispatch();
}
}
threads.h
#ifndef THREADS_H_
#define THREADS_H_
#include <stddef.h>
#include <setjmp.h>
#include <stdbool.h>
struct thread {
struct thread* next;
void (*f)(void*);
void* arg;
void* stack_ptr;
size_t stack_size;
bool has_run;
size_t rbp;
jmp_buf jmp_buf;
};
struct thread* thread_create(void (*f)(void*), void* arg);
void thread_queue(struct thread* t);
void thread_yield(void);
void thread_exec(void);
static void dispatch(void);
static void schedule(void);
#endif // THREADS_H_
main.c
#include <stdio.h>
#include <stdlib.h>
#include "threads.h"
static int a3 = 3;
void arg_printer(void* arg) {
int id = *(int*)arg;
if (id == 1) {
thread_queue(thread_create(arg_printer, &a3));
}
for (size_t i = 0; i < 3; ++i) {
printf("printing from %i, now yielding\n", id);
thread_yield();
}
}
int main() {
printf("started at main\n");
int a1 = 1;
int a2 = 2;
thread_queue(thread_create(arg_printer, &a1));
thread_queue(thread_create(arg_printer, &a2));
thread_exec();
printf("finished at main\n");
return 0;
}
For some reason, when running with compiler optimizations, I get the following error:
*** longjmp causes uninitialized stack frame ***: terminated
Someone who also encountered such a problem put it this way:
The problem is that the "longjmp causes uninitialized stack frame" is really a false positive. I'm abusing the stack in ways beyond the imagination of glibc.
The "fortified" (checked) longjmp doesn't take heap-allocated stacks into account; after all, _FORTIFY_SOURCE is to perform checks of standard conforming programs.
I'd like to get this working without disabling compiler inserted safety checks and would appreciate any suggestions.
You don't have to disable _FORTIFY_SOURCE everywhere; your program works if you just disable it for the threads.c compilation unit, where the critical longjmp is.